A simple augmented reality application

原創

2020-06-16 14:35

主要理解代碼 + 記錄解決報錯問題，具體內容有空補上。

相機矩陣轉換成 OpenGL 格式

在 OpenGL 中主要使用 4x4 矩陣來表示轉換，這個和 3x4 的相機矩陣不同。然而，OpenGL 中的 GL_PROJECTION 和 GL_MODELVIEW 是將相機矩陣分開來表示。其中 GL_PROJECTION 表示相機的內參數 K 矩陣；GL_MODELVIEW 表示物體和相機之間的轉換關係，可以粗略地表示爲 R 和 t 矩陣。

代碼如下所示， K 表示校準後的相機內參數矩陣。

from OpenGL.GL import *
from OpenGL.GLU import *
import pygame, pygame.image
from pygame.locals import *
from OpenGL.GLUT import *
import numpy as np
from scipy import linalg
import sys


def set_projection_from_camera(K):
    
    """
    Set view from a camera calibration matrix.
    """
    
    glMatrixMode(GL_PROJECTION)
    glLoadIdentity()
    
    fx = K[0, 0]
    fy = K[1, 1]
    # 計算以度爲單位的垂直視野
    fovy = 2*np.arctan(0.5*height/fy)*180/np.pi
    # 縱橫比
    aspect = (width*fy)/(height*fx)
    
    # define the near and far clipping planes
    near = 0.1
    far = 100.0
    
    # set perspective
    gluPerspective(fovy, aspect, near, far)
    glViewport(0, 0, width, height)

P = K[R|t]，所以得到 Rt = PK^-1 。Rt 爲下面代碼的輸入：


def set_modelview_from_camera(Rt):
    
    """
    Set the model view matrix from camera pose.
    """
    glMatrixMode(GL_MODELVIEW)
    glLoadIdentity()
    
    # rotate teapot 90 deg around x-axis so that z-axis is up
    Rx = np.array([[1, 0, 0], [0, 0, -1], [0, 1, 0]])
    
    # set rotation to best approximation
    R = Rt[:, :3]
    U, S, V = linalg.svd(R)
    R = np.dot(U, V)
    R[0, :] = -R[0, :] # change sign of x-axis
    
    # set translation
    t = Rt[:, 3]
    
    # set 4*4 model view matrix
    M = np.eye(4)
    M[:3, :3] = np.dot(R, Rx)
    M[:3, 3] = t
    
    # transpose and flatten to grt column order
    M = M.T
    m = M.flatten()
    
    # replace model view with new matrix
    glLoadMatrixf(m)

將虛擬物體投放到圖片上

def draw_background(imname):
    
    """
    Draw background image using a quadrilateral.
    """
    
    # load backgound image (should be .bmp) to OpenGL texture
    bg_image = pygame.image.load(imname).convert()
    bg_data = pygame.image.tostring(bg_image, "RGBX", 1)
    
    glMatrixMode(GL_MODELVIEW)
    glLoadIdentity()
    glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
    
    # bind the texture
    glEnable(GL_TEXTURE_2D)
    glBindTexture(GL_TEXTURE_2D, glGenTextures(1))
    glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, width, height, 0, GL_RGBA, GL_UNSIGNED_BYTE, bg_data)
    glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
    glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
    
    # create quad to fill the whole window
    glBegin(GL_QUADS)
    glTexCoord2f(0.0, 0.0); glVertex3f(-1.0, -1.0, -1.0)
    glTexCoord2f(1.0, 0.0); glVertex3f( 1.0, -1.0, -1.0)
    glTexCoord2f(1.0, 1.0); glVertex3f( 1.0,  1.0, -1.0)
    glTexCoord2f(0.0, 1.0); glVertex3f(-1.0,  1.0, -1.0)
    glEnd()
    
    # clear the texture
    glDeleteTextures(1)

def draw_teapot(size):
    
    """
    Draw a red teapot at the origin.
    """
    glEnable(GL_LIGHTING) 
    glEnable(GL_LIGHT0) 
    glEnable(GL_DEPTH_TEST) 
    glClear(GL_DEPTH_BUFFER_BIT)
    
    # draw red teapot
    glMaterialfv(GL_FRONT,GL_AMBIENT,[0,0,0,0]) 
    glMaterialfv(GL_FRONT,GL_DIFFUSE,[0.5,0.0,0.0,0.0]) 
    glMaterialfv(GL_FRONT,GL_SPECULAR,[0.7,0.6,0.6,0.0]) 
    glMaterialf(GL_FRONT,GL_SHININESS,0.25*128.0) 
    glutSolidTeapot(size)

主代碼：

from OpenGL.GL import *
from OpenGL.GLU import *
from OpenGL.GLUT import *
import pygame, pygame.image
from pygame.locals import *
import pickle

width, height = 1000, 747

def setup():
    
    """
    Setup window and pygame environment.
    """
    
    pygame.init()
    pygame.display.set_mode((width, height), OPENGL | DOUBLEBUF)
    pygame.display.set_caption('OpenGL AR demo')
    
# load camera data
with open('ar_camera.pkl', 'rb') as f:
    K = pickle.load(f)
    Rt = pickle.load(f)
    
setup()
draw_background('./book_perspective.bmp')
set_projection_from_camera(K)
set_modelview_from_camera(Rt)
draw_teapot(0.05)

pygame.display.flip()
while True:
  for event in pygame.event.get():
    if event.type == pygame.QUIT:
       sys.exit()
       pygame.quit()
pygame.display.flip()

結果：