Edit: got this working. Here's code to project world coords to normalize screen coords [-1, 1] so you can then feed them into h3dShowOverlay to get billboards.
Code:
void h3dutProject(H3DNode cameraNode, Vec3f world, float *x, float *y) {
Matrix4f projMat;
h3dGetCameraProjMat(cameraNode, projMat.x);
const float *camTrans;
h3dGetNodeTransMats( cameraNode, 0x0, &camTrans );
Matrix4f viewMat( camTrans );
viewMat = viewMat.inverted();
Matrix4f viewProjMat = projMat * viewMat;
Vec3f screen = viewProjMat * world;
if (x) {
*x = screen.x / screen.z;
}
if (y) {
*y = screen.y / screen.z;
}
}