Function ConvertProjectionCoordToWorldPickRay

trinity/TriMath.cpp:1070–1102

bool ConvertProjectionCoordToWorldPickRay( float x, float y, const Matrix* projMat, const Matrix* viewMat, Vector3* rayStart, Vector3* rayDir )
{
	Matrix projection2view;
	Matrix view2world;

	if( !Inverse( projection2view, *projMat ) )
	{
		return false;
	}

	if( !Inverse( view2world, *viewMat ) )
	{
		return false;
	}

	// Initialize 'rayStart' with projection coordinates and transform it to world
	rayStart->x = x;
	rayStart->y = y;
	rayStart->z = 0.0f;

	// Note that this method does _unprojection_ i.e. divides by w
	*rayStart = TransformCoord( *rayStart, projection2view );
	*rayStart = TransformCoord( *rayStart, view2world );

	// Initialize 'rayEnd' with projection coordinates and put it halfway into projection space
	Vector3 rayEnd( x, y, 0.5f );
	rayEnd = TransformCoord( rayEnd, projection2view );
	rayEnd = TransformCoord( rayEnd, view2world );

	*rayDir = Normalize( rayEnd - *rayStart );

	return true;
}