simplAR: Augmented reality for OpenCV beginners

Here is a simple AR demo for beginners (-includes me). The program augments only in 2D with a picture or a clip.
The video plays rather slow, it would be great if you could use multiple threads (-which i have no clue what so ever!).


*Update - Download links fixed!
Download the Project folder and the source code [6.62 Mb].


Links:
http://www.bigbuckbunny.org/
If you want to download only the (modified) video used in this demo, try the link below-
BigBuckBunny_Trailer [6.43 Mb]
640x480, Xvid/LameMP3 


Pattern (Size A4,JPEG):
















Videos:




Source Code:
//______________________________________________________________________________________
// OpenCV Simple Augmented Reality Program
// Author: Bharath Prabhuswamy
//______________________________________________________________________________________
//______________________________________________________________________________________
#include <stdio.h>
#include <stdlib.h>
#include "cv.h"
#include "highgui.h"

int main()
{
    CvCapture *capture = 0;
    IplImage  *image = 0;
    IplImage *frame = 0;
    IplImage *disp,*neg_img,*cpy_img;
    int key = 0;
    int fcount = 0;
    int option = 0;
 
    capture = cvCaptureFromCAM( 0 );
 if ( !capture ) 
        return -1;

    //Use a video with aspect ratio 4:3
    CvCapture* vid = cvCreateFileCapture("trailer.avi");
    if ( !vid )
       return -1;

    IplImage *pic = cvLoadImage("pic.jpg");
    cvFlip(pic,pic,1);
 
 int b_width  = 5;
 int b_height = 4;
 int b_squares = 20;
 CvSize b_size = cvSize( b_width, b_height );
 //The pattern actually has 6 x 5 squares, but has 5 x 4 = 20 'ENCLOSED' corners

 CvMat* warp_matrix = cvCreateMat(3,3,CV_32FC1);
 CvPoint2D32f* corners = new CvPoint2D32f[ b_squares ];
 int corner_count;

 printf("Select an option to run the program\n\n");
 printf("1. Show an Image over the pattern.\n");
 printf("2. Play a Clip over the pattern.\n");
 printf("3. Mark the pattern.\n\n");
 scanf("%d",&option);
  
 //Quit on invalid entry
 if(!(option>=1 && option<=3))
 {
  printf("Invalid selection.");
  return -1;
 }

 cvNamedWindow("Video",CV_WINDOW_AUTOSIZE);

 while(key!='q') 
 {
  image = cvQueryFrame( capture );
  if( !image ) break;
  cvFlip(image,image,1);

  disp = cvCreateImage( cvGetSize(image), 8, 3 );
  cpy_img = cvCreateImage( cvGetSize(image), 8, 3 );
                neg_img = cvCreateImage( cvGetSize(image), 8, 3 );

  IplImage* gray = cvCreateImage( cvGetSize(image), image->depth, 1);
  int found = cvFindChessboardCorners(image, b_size, corners, &corner_count,         
                                    CV_CALIB_CB_ADAPTIVE_THRESH | CV_CALIB_CB_FILTER_QUADS);

  cvCvtColor(image, gray, CV_BGR2GRAY);
  
  //This function identifies the pattern from the gray image, saves the valid group of corners
  cvFindCornerSubPix(gray, corners, corner_count,  cvSize(11,11),cvSize(-1,-1),     
                                    cvTermCriteria(CV_TERMCRIT_EPS+CV_TERMCRIT_ITER, 30, 0.1 ));
   
  if( corner_count == b_squares ) 
         {
   if(option == 1)
   {
    CvPoint2D32f p[4];
    CvPoint2D32f q[4];

    IplImage* blank  = cvCreateImage( cvGetSize(pic), 8, 3);
    cvZero(blank);
    cvNot(blank,blank);
    
    //Set of source points to calculate Perspective matrix
    q[0].x= (float) pic->width * 0;
    q[0].y= (float) pic->height * 0;
    q[1].x= (float) pic->width;
    q[1].y= (float) pic->height * 0;

    q[2].x= (float) pic->width;
    q[2].y= (float) pic->height;
    q[3].x= (float) pic->width * 0;
    q[3].y= (float) pic->height;
  
    //Set of destination points to calculate Perspective matrix
    p[0].x= corners[0].x;
    p[0].y= corners[0].y;
    p[1].x= corners[4].x;
    p[1].y= corners[4].y;
    
    p[2].x= corners[19].x;
    p[2].y= corners[19].y;
    p[3].x= corners[15].x;
    p[3].y= corners[15].y;
    
    //Calculate Perspective matrix
    cvGetPerspectiveTransform(q,p,warp_matrix);

    //Boolean juggle to obtain 2D-Augmentation
    cvZero(neg_img);
    cvZero(cpy_img);

    cvWarpPerspective( pic, neg_img, warp_matrix);
    cvWarpPerspective( blank, cpy_img, warp_matrix);
    cvNot(cpy_img,cpy_img);

    cvAnd(cpy_img,image,cpy_img);
    cvOr(cpy_img,neg_img,image);

    cvShowImage( "Video", image); 
   }
   else if(option == 2)
   {
    CvPoint2D32f p[4];
    CvPoint2D32f q[4];

    frame = cvQueryFrame(vid);
    if (!frame)
    printf("error frame");

    IplImage* blank  = cvCreateImage( cvGetSize(frame), 8, 3);
    cvZero(blank);
    cvNot(blank,blank);

    q[0].x= (float) frame->width * 0;
    q[0].y= (float) frame->height * 0;
    q[1].x= (float) frame->width;
    q[1].y= (float) frame->height * 0;

    q[2].x= (float) frame->width;
    q[2].y= (float) frame->height;
    q[3].x= (float) frame->width * 0;
    q[3].y= (float) frame->height;
  
    p[0].x= corners[0].x;
    p[0].y= corners[0].y;
    p[1].x= corners[4].x;
    p[1].y= corners[4].y;
    
    p[2].x= corners[19].x;
    p[2].y= corners[19].y;
    p[3].x= corners[15].x;
    p[3].y= corners[15].y;
    
    cvGetPerspectiveTransform(q,p,warp_matrix);

    //Boolean juggle to obtain 2D-Augmentation
    cvZero(neg_img);
    cvZero(cpy_img);

    cvWarpPerspective( frame, neg_img, warp_matrix);
    cvWarpPerspective( blank, cpy_img, warp_matrix);
    cvNot(cpy_img,cpy_img);

    cvAnd(cpy_img,image,cpy_img);
    cvOr(cpy_img,neg_img,image);

    cvShowImage( "Video", image); 
   }
   else
   {
    CvPoint p[4];

    p[0].x=(int)corners[0].x;
    p[0].y=(int)corners[0].y;
    p[1].x=(int)corners[4].x;
    p[1].y=(int)corners[4].y;
    
    p[2].x=(int)corners[19].x;
    p[2].y=(int)corners[19].y;
    p[3].x=(int)corners[15].x;
    p[3].y=(int)corners[15].y;
    
    cvLine( image, p[0], p[1], CV_RGB(255,0,0),2);
    cvLine( image, p[1], p[2], CV_RGB(0,255,0),2);
    cvLine( image, p[2], p[3], CV_RGB(0,0,255),2);
    cvLine( image, p[3], p[0], CV_RGB(255,255,0),2);

    //or simply
    //cvDrawChessboardCorners(image, b_size, corners, corner_count, found);
    
    cvShowImage( "Video", image); 
   }
  }
  else
  {
   //Show gray image when pattern is not detected
   cvFlip(gray,gray);
   cvShowImage( "Video", gray );

  }
  key = cvWaitKey(1);
  
 }

    cvDestroyWindow( "Video" );
    cvReleaseCapture( &vid );
    cvReleaseMat(&warp_matrix);
    cvReleaseCapture( &capture );

    return 0;
}

19 comments:

  1. Hi! I haven't tried running this, but what will happen if I tilt the pattern towards or away from the camera (about an axis parallel to the cam)?

    ReplyDelete
  2. As long as the pattern is visible, the picture gets augmented (up to certain acute angle and distance).

    ReplyDelete
  3. great sample.... thanks 4 sharing..... :)

    ReplyDelete
  4. While the sample works, the coding is horrible.
    In your example you have multiple memory leaks - when you create images using cvCreateImage, images are never released. Compiler will not automatically free this memory for you, so the application will run out of memory and crash. This isn't java or python, if you want memory to be automatically released, use smart pointers.

    ReplyDelete
  5. @segfault: True, here's an article that might help - http://www.aishack.in/2010/01/opencv-memory-management

    ReplyDelete
  6. @segfault: I agree that this code is Horrible.

    I quote what I said before..."These programs are just guidelines of the algorithms that help you achieve the objective of the program."

    ..And its up to you to go ahead and show all the programming skills, having done this.

    ReplyDelete
  7. hi bharath is there any sample with 3d animation with c# & open cv

    ReplyDelete
  8. hi,

    I'm wondering if it would be possible to track multiple markers using opencv? This is a very cool sample by the way!

    ReplyDelete
  9. @tanarama:

    I ve not heard of any 3D animation using opencv, i would suggest combining opengl to get 3d objects in to the frame.

    @Eamonn:

    Thanks..:)..
    Yes it is possible to have multiple marker tracking system. The above program and any other program for that matter which uses the function
    cvFindChessboardCorners detects only one pattern. if two such patterns are shown it would be confused and throw an error.

    The only way to detect more than one pattern is to have a custom pattern detecting algorithm.

    ReplyDelete
  10. thanks for the info, do you have any helpful links where I could start to do this?

    ReplyDelete
  11. Its a good idea to have a copy of Opencv O'reilly book.
    http://oreilly.com/catalog/9780596516130

    You can look into ARToolkit, an open source lib. for augmented reality. It can track multiple markers and augment it.
    http://www.hitl.washington.edu/artoolkit/documentation/

    ReplyDelete
  12. Is there a way to do multiple marker tracking? thanks

    ReplyDelete
  13. Hi Bharath,
    thank you for the wonderful example.
    But I am having trouble compiling the source file, I am getting this error:

    error LNK2019: unresolved external symbol _cvFindChessboardCorners referenced in function _main

    I have included almost all files, which header file I might be missing?

    ReplyDelete
  14. Thanks the program is working perfectly (QT 4.7 and OpenCV 2.1), just would be better if you use some "if" when the pointer to images and videos is empty.

    Bye!

    ReplyDelete
  15. Hey awesome work buddy..similarly is it possible to overlay 3d models on to the pattern ..if yes then pls do share

    ReplyDelete
  16. Gr8! Works out of the box - Used it on Ubuntu !

    ReplyDelete
  17. Thanks a lot, it worked very well
    I was trying to find a simple marker tracker in opencv !!!

    Roger

    ReplyDelete
  18. hello.

    nice post..

    i want to implement same with android..can u please guide for the same.

    ReplyDelete