Кажется, что переменные modelview1 и projection1 имеют неправильный тип.
Я нашел следующий фрагмент (здесь http://www.3dkingdoms.com/selection.html):
// This function will find 2 points in world space that are on the line into the screen defined by screen-space( ie. window-space ) point (x,y)
double mvmatrix[16];
double projmatrix[16];
int viewport[4];
double dX, dY, dZ, dClickY; // glUnProject uses doubles, but I'm using floats for these 3D vectors
glGetIntegerv(GL_VIEWPORT, viewport);
glGetDoublev (GL_MODELVIEW_MATRIX, mvmatrix);
glGetDoublev (GL_PROJECTION_MATRIX, projmatrix);
dClickY = double (g_WindowHeight - y); // OpenGL renders with (0,0) on bottom, mouse reports with (0,0) on top
gluUnProject ((double) x, dClickY, 0.0, mvmatrix, projmatrix, viewport, &dX, &dY, &dZ);
ClickRayP1 = Vector3 ( (float) dX, (float) dY, (float) dZ );
gluUnProject ((double) x, dClickY, 1.0, mvmatrix, projmatrix, viewport, &dX, &dY, &dZ);
ClickRayP2 = Vector3 ( (float) dX, (float) dY, (float) dZ );