QuickTime


	Log In \| Not a Member?	Contact ADC

Computing the 3x3 Matrix to Composite a Movie or Image into QuickTime VR's Pre-Screen Buffer

Dispatch 23

QuickTime 4 has been enhanced to include a more accurate renderer in QuickTime VR. With this, it is possible to embed an image or movie in three-space and transform it in such a way that it appears to be locked to the QuickTime VR panorama as you pan and zoom.

While at this time, there is no data driven support for displaying movies within panoramas, it is possible to write an application that transforms the movie and composites it into the QuickTime VR pre-screen buffer.

The key to doing this correctly is to accurately determine, from the viewing and placement parameters, the 3x3 matrix used to transform the image. Performing the image transformation itself is left as an exercise for the reader, but there are many ways to do this, including any 3D renderer capable of texture-mapping and QuickTime itself.

The process is shown in the illustration below:

Figure 1.

First, QuickTime VR dewarps the cylinder according to the current view parameters. Then the current movie frame is transformed in perspective according to the current view parameters. Then the composite result is copied to the window.

Samples of a panning sequence are shown below.

In the code below, we provide two interfaces: ImmerseImageInQTVR() and ImmerseQTImageInQTVR(). The first produces a floating-point 3x3 matrix, and the second produces a fixed-point 3x3 matrix of the type used in QuickTime.

The parameters for the image and the view are the same: { GWorld, pan, tilt, fov }. Given these parameters, we determine the transformation to take the GWorld into 3D. By inverting one of these transformations, and concatenating, we achieve the transformation to take one GWorld into the other.

Using the Resulting Matrices

The matrices used with the QuickTime are fixed point and can easily represent milder forms of perspective. For more extreme perspectives, it may be necessary to use a floating point matrix and custom rendering code. Code to generate both fixed and floating point matrices is provided below.

#include <fp.h>
#include <Movies.h>

typedef float vec3[3];

static float FOVToFocalLength(float fov, long height)
{
	return((height - 1) * 0.5F / tan(fov * 0.5F));
}

static float FocalLengthToFOV(float focalLength, long height)
{
	return(2.0F * atan((height - 1) * 0.5F / focalLength));
}

static void SetRotationX3x3(float rad, float *M) 
{
	register vec3* m = (vec3*)M;
	float s = sin(rad); float c = cos(rad);
	m[0][0] = 1;	m[0][1] = 0;	m[0][2] = 0;
	m[1][0] = 0;	m[1][1] = c;	m[1][2] = s;
	m[2][0] = 0;	m[2][1] = -s;	m[2][2] = c;
}

static void SetRotationY3x3(float rad, float *M)
{
	register vec3* m = (vec3*)M;
	float s = sin(rad); float c = cos(rad);
	m[0][0] = c;	m[0][1] = 0;	m[0][2] = -s;
	m[1][0] = 0;	m[1][1] = 1;	m[1][2] = 0;
	m[2][0] = s;	m[2][1] = 0;	m[2][2] = c;
}

static void ScaleMatrixNxM(register float scale, register float *m, long nRows, long nCols)
{
	register long i = nRows * nCols;
	
	for ( ; i--; m++)
		*m *= scale;
}

static void LinearTransform(
	const float            *A,	/* nxm */
	const float            *B,	/* mxp */
	register float         *C,	/* nxp */
	register unsigned long n,
	unsigned long          m,
	unsigned long          p
)
{
	register const float *ap, *bp;
	register unsigned long k, j, i;
	register double_t sum;		/* Extended precision for intermediate results */

	for (i = n; i--; A += m) {	/* Each row in A */
		for (j = 0; j < p; j++) {	/* Each column in B */
			ap = A;			/* Left of ith row of A */
			bp = B + j;		/* Top of jth column of B */
			sum = 0.0;
			for (k = m; k--; bp += p)
				sum += *ap++ * (*bp);   /* *C += A[i'][k'] * B[k'][j]; */
			*C++ = sum;
		}
	}
}

static void ComputeProjectionMatrix3x3(
	unsigned long	width,	/* Image width */
	unsigned long	height,	/* Image height */
	float	dist,	/* Distance to the image */
	float	copX,	/* Center of projection, relative to center of image */
	float	copY,	/* Center of projection, relative to center of image */
	float	*M	/* Projection matrix, from 3D to 2D */
)
{
	float cx, cy;
	
	cx = (width  - 1) * 0.5f + copX;
	cy = (height - 1) * 0.5f + copY;
	
	M[0*3+0] = -dist;	M[0*3+1] = 0.0f;	M[0*3+2] = 0.0f;
	M[1*3+0] = 0.0f;	M[1*3+1] = dist;	M[1*3+2] = 0.0f;
	M[2*3+0] = cx;		M[2*3+1] = cy;		M[2*3+2] = 1.0f;
}

static void ComputeImmersionMatrix3x3(
	unsigned long	width,	/* Image width */
	unsigned long	height,	/* Image height */
	float	dist,	/* Distance to the image */
	float	copX,	/* Center of projection, relative to center of image */
	float	copY,	/* Center of projection, relative to center of image */
	float	*M	/* Immersion matrix,  from 2D to 3D */
)
{
	float cx, cy, dinv;
	
	/* Compute projection matrix parameters */
	dinv = 1.0f / dist;
	cx = ((width  - 1) * 0.5f + copX) * dinv;
	cy = ((height - 1) * 0.5f + copY) * dinv;

	M[0*3+0] = -dinv;	M[0*3+1] = 0.0f;	M[0*3+2] = 0.0f;
	M[1*3+0] = 0.0f;	M[1*3+1] = dinv;	M[1*3+2] = 0.0f;
	M[2*3+0] = cx;		M[2*3+1] = -cy;		M[2*3+2] = 1.0f;
}

static void PlaceImageViewRot3x3(
	unsigned long	width,	/* Image width */
	unsigned long	height,	/* Image height */
	float	dist,	/* Distance to the image */
	const float	*Ri,	/* Rotation matrix from 3D to 2D */
	const float	*cop,	/* 2-vector center of projection, relative to center of image, in image coordinates (0,0 if NULL) */
	float	*M,	/* The resultant matrix (if non-NULL) */
	float	*Mi	/* and its inverse matrix (if non-NULL) */
)
{
	float C[3][3];
	float cx, cy;
	
	if (cop != NULL) {
		cx = cop[0];
		cy = cop[1];
	}
	else {
		cx = 0;
		cy = 0;
	}

	if (Mi != NULL) {	/* From 3D to 2D space */
		ComputeProjectionMatrix3x3(width, height, dist, cx, cy, C[0]);
		LinearTransform(Ri, C[0], Mi, 3, 3, 3);
	}

	if (M != 0) {	/* From 2D to 3D space */
		float R[3][3];
		R[0][0] = Ri[0];	R[0][1] = Ri[3];	R[0][2] = Ri[6];
		R[1][0] = Ri[1];	R[1][1] = Ri[4];	R[1][2] = Ri[7];
		R[2][0] = Ri[2];	R[2][1] = Ri[5];	R[2][2] = Ri[8];
		ComputeImmersionMatrix3x3(width, height, dist, cx, cy, C[0]);
		LinearTransform(C[0], R[0], M, 3, 3, 3);
	}
}

static void PlaceImagePanTiltFOV3x3(
	unsigned long	width,	/* Image width */
	unsigned long	height,	/* Image height */
	float	pan,	/* Pan */
	float	tilt,	/* Tilt */
	float	fov,	/* Vertical field of view */
	const float	*cop,	/* 2-vector center of projection, relative to center of image */
	float	*M,	/* The resultant matrix (if non-NULL) */
	float	*Mi	/* and its inverse matrix (if non-NULL) */
)
{
	float R[3][3], P[3][3], T[3][3];
	float dist;
	
	SetRotationY3x3(-pan,  P[0]);
	SetRotationX3x3(-tilt, T[0]);
	LinearTransform(P[0], T[0], R[0], 3, 3, 3);
	dist = FOVToFocalLength(fov, height);
	PlaceImageViewRot3x3(width, height, dist, R[0], cop, M, Mi);
}

static void MakeQTMatrix(float F[3][3], MatrixRecord *Q)
{
	float x[3][3], H[3][3], G[3][3];
	float max, t;
	
	x[0][0] = 1;	x[0][1] = 0;	x[0][2] = 0;
	x[1][0] = 0;	x[1][1] = 1;	x[1][2] = 0;
	x[2][0] = -0.5;	x[2][1] = -0.5;	x[2][2] = 1;
	LinearTransform(x[0], F[0], H[0], 3, 3, 3);
	x[2][0] = 0.5;	x[2][1] = 0.5;
	LinearTransform(H[0], x[0], G[0], 3, 3, 3);
	
	max = fabs(G[0][2]);
	if (max < (t = fabs(G[1][2])))
		max = t;
	if (max < (t = fabs(G[2][2])))
		max = t;
	if (max > 1) 
		ScaleMatrixNxM(1/max, G[0], 3, 3);
	Q->matrix[0][0] = G[0][0] * 65536.0f;
	Q->matrix[0][1] = G[0][1] * 65536.0f;
	Q->matrix[0][2] = G[0][2] * 1073741824.0f;
	Q->matrix[1][0] = G[1][0] * 65536.0f;
	Q->matrix[1][1] = G[1][1] * 65536.0f;
	Q->matrix[1][2] = G[1][2] * 1073741824.0f;
	Q->matrix[2][0] = G[2][0] * 65536.0f;
	Q->matrix[2][1] = G[2][1] * 65536.0f;
	Q->matrix[2][2] = G[2][2] * 1073741824.0f;
}

void ImmerseImageInQTVR(
	GWorldPtr	imageGW,
	float		imagePan,
	float		imageTilt,
	float		imageFOV,
	GWorldPtr	viewGW,
	float		viewPan,
	float		viewTilt,
	float		viewFOV,
	float		*M
)
{
	Rect r;
	unsigned long width, height;
	float V[3][3], P[3][3];
	
	r = imageGW->portRect;
	width  = r.right  - r.left;
	height = r.bottom - r.top;
	PlaceImagePanTiltFOV3x3(width, height, imagePan, imageTilt, imageFOV, NULL, P[0], NULL);
	
	r = viewGW->portRect;
	width  = r.right  - r.left;
	height = r.bottom - r.top;
	PlaceImagePanTiltFOV3x3(width, height, viewPan,  viewTilt,  viewFOV,  NULL, NULL, V[0]);

	LinearTransform(P[0], V[0], M, 3, 3, 3);
}

void ImmerseQTImageInQTVR(
	GWorldPtr	imageGW,
	float		imagePan,
	float		imageTilt,
	float		imageFOV,
	GWorldPtr	viewGW,
	float		viewPan,
	float		viewTilt,
	float		viewFOV,
	MatrixRecord *Q
)
{
	float M[3][3];

	ImmerseImageInQTVR(imageGW, imagePan, imageTilt, imageFOV, viewGW, viewPan, viewTilt, viewFOV, M[0]);
	MakeQTMatrix(M, Q);
}

Change History

2/23/00 - ket - First published

Topics
Previous | Next

Computing the 3x3 Matrix to Composite a Movie or Image into QuickTime VR's Pre-Screen Buffer

Dispatch 23

Using the Resulting Matrices

See Also

Change History