For NPC AI to work they have to have simulated vision. This is done by using raycast/linepick.

My vision system does the following:-

Takes into account head position - using head bone

Takes into account rotation of NPC spine bone (used for aiming)

Build list of enemies in view range

Takes into account FOV

It looks through each of the ememies bones to see if any part of the enemy is visible. It returns location of the first bone found. This is to make sure enemies are still seen even if they are partially obscured. It has a render function to help with debugging the AI.

Here's the code: -

#ifndef VISION_H
#define VISION_H

#include "gamelib/include/gamelib.h"
#include "gamelib/include/scene.h"
#include "actor/include/animBody.h"

typedef enum SeenState
{
SEEN_IDLE,
SEEN_ALERT,
SEEN_ACTIVE
};

typedef enum visionMode
{
VISION_BONE,
VISION_BOUNDING,
VISION_NAVMESH
};

//
// Implement actor vision
//
class Vision
{
protected:
TScene *m_scene;
bodyControl *m_body;
float m_lastCheckTime;
float m_lastSeenTime;
float m_notSeenTime;
float m_lockTime;
float m_updateRate;
float m_idleTime;
bool m_seen;
float m_viewRange;
float m_viewAngle;
int m_cnt;
public:
SeenState m_state;
map<float,TEntity> m_nearbyEnemy;
TEntity m_entity, m_lastTarget, m_pivot, m_head, m_target;
string m_team;
TPick m_pick;
TVec3 m_lastSeenPos, m_targetPos;
Vision(TScene *s, TEntity e, TEntity head, bodyControl *pbody);

bool canSeeEnemy(visionMode mode= VISION_BONE);
bool canSeeEntity(TEntity target, visionMode mode = VISION_BONE);
bool canSeeBoundBox(TVec3 pos, TEntity target);
bool canSeeBone( TVec3 pos, TEntity target);
TVec3 getAimAtLocation();

bool navMeshCanSeeEntity(TEntity target);
inline TEntity getLastTarget() { return m_lastTarget; } 
inline SeenState Vision::getSeenState() { return m_state; }
float distanceToEnemy();

void Render(TCamera cam);
};

#endif

#include "actor/include/vision.h"
#include "gamelib/include/gamemath.h"
//#define DEBUG_VISION

Vision::Vision( TScene *s, TEntity e, TEntity h, bodyControl *pBody ) : m_lastCheckTime(0), m_updateRate(100), m_lastTarget(NULL), m_seen(false),
		m_viewRange(50), m_viewAngle(180), m_cnt(0), m_lockTime(5000), m_idleTime(60000)
{
m_entity = e;
m_scene = s;
m_cnt = 0;
m_lastSeenTime = 0;
m_seen = false;
m_state = SEEN_IDLE;
m_pivot = CreatePivot();
EntityParent( m_pivot, m_entity );
m_body = pBody;
m_head = h;
m_notSeenTime = AppTime();
m_team = GetEntityKey(e,"team");
}

bool Vision::navMeshCanSeeEntity(TEntity target)
{
static const int MAX_POLYS = 256;
dtStatPolyRef m_startRef;
dtStatPolyRef m_endRef;
dtStatPolyRef m_polys[MAX_POLYS];
dtStatPolyRef m_parent[MAX_POLYS];
float m_spos[3], m_epos[3];
int m_npolys;
float m_polyPickExt[3];
float wallDist, navhit;

float elapse = AppTime() - m_lastCheckTime;
if((target == m_lastTarget) && (elapse < m_updateRate))
	return m_seen;

m_targetPos = EntityPosition( target, 1);
m_lastCheckTime = AppTime();
m_lastTarget = target;
m_cnt += 1;

m_seen = false;

float dist = EntityDistance(m_entity, target);
// check If the target is within viewrange
if (dist<=m_viewRange)
{
	// observer vector
	PositionEntity( m_pivot, EntityPosition( m_head,1),1);
	TVec3 tform = TFormVector( Vec3(0,0,1), m_body->pivot, NULL);
	AlignToVector(m_pivot, tform, 3, 1);

	TVec3 observerPos = EntityPosition( m_entity, 1);
	// pick vector
	float angle = angleToDest( m_pivot, m_targetPos );
	if(angle <= (m_viewAngle/2.0))
	{
		// check If something is blocking the view
		m_spos[0] = observerPos.X*-1;
		m_spos[1] = observerPos.Y;
		m_spos[2] = observerPos.Z;
		m_epos[0] = m_targetPos.X*-1;
		m_epos[1] = m_targetPos.Y;
		m_epos[2] = m_targetPos.Z;
		m_polyPickExt[0] = 2;
		m_polyPickExt[1] = 4;
		m_polyPickExt[2] = 2;

		m_startRef = m_scene->navmesh->m_navMesh->findNearestPoly(m_spos, m_polyPickExt);
		m_npolys = m_scene->navmesh->m_navMesh->raycast(m_startRef, m_spos, m_epos, navhit, m_polys, MAX_POLYS);

		if (navhit >=1)
		{
			m_seen = true;
			m_lastTarget = target;
			m_lastSeenPos = m_targetPos;
			m_lastSeenTime = AppTime();
			m_state = SEEN_ACTIVE;
		}
	}
}
// observer cannot see target
if (!m_seen)
{
	if ((AppTime() - m_lastSeenTime) < this->m_lockTime)
		m_lastSeenPos = EntityPosition( target, 1);
	m_lastTarget = NULL;
	if ((AppTime() - m_lastSeenTime) > m_idleTime)
		m_state = SEEN_IDLE;
	else
		m_state = SEEN_ACTIVE;
}
else if (m_seen)	// Cheat if it's with locked on time
{
	m_lastTarget = target;
	m_lastSeenPos = EntityPosition( target, 1);
}
return m_seen;
}

TEntity pickSource, pickDest;

int _stdcall filter( TEntity entity ) 
{
if ( isParent(pickSource, entity) ) return 0;

return 1;
}

bool Vision::canSeeBoundBox( TVec3 pos, TEntity target)
{
TVec6 box;
float margin = 0.3;
box = GetEntityAABB( target );
box.X0 += margin;
box.Y0 += margin;
box.Z0 += margin;
box.X1 -= margin;
box.Y1 -= margin;
box.Z1 -= margin;
if (!LinePick( &m_pick, pos, Vec3(box.X0, box.Y0, box.Z0), 0.1,0, (BP)filter )) return true;
if (!m_pick.entity || isParent(m_target, m_pick.entity )) return true;

if (!LinePick( &m_pick, pos, Vec3(box.X0, box.Y1, box.Z0), 0.1,0, (BP)filter )) return true;
if (!m_pick.entity || isParent(m_target, m_pick.entity )) return true;

if (!LinePick( &m_pick, pos, Vec3(box.X1, box.Y0, box.Z0), 0.1,0, (BP)filter )) return true;
if (!m_pick.entity || isParent(m_target, m_pick.entity )) return true;

if (!LinePick( &m_pick, pos, Vec3(box.X1, box.Y1, box.Z0), 0.1,0, (BP)filter )) return true;
if (!m_pick.entity || isParent(m_target, m_pick.entity )) return true;


if (!LinePick( &m_pick, pos, Vec3(box.X0, box.Y0, box.Z1), 0.1,0, (BP)filter )) return true;
if (!m_pick.entity || isParent(m_target, m_pick.entity )) return true;

if (!LinePick( &m_pick, pos, Vec3(box.X0, box.Y1, box.Z1), 0.1,0, (BP)filter )) return true;
if (!m_pick.entity || isParent(m_target, m_pick.entity )) return true;

if (!LinePick( &m_pick, pos, Vec3(box.X1, box.Y0, box.Z1), 0.1,0, (BP)filter )) return true;
if (!m_pick.entity || isParent(m_target, m_pick.entity )) return true;

if (!LinePick( &m_pick, pos, Vec3(box.X1, box.Y1, box.Z1), 0.1,0, (BP)filter )) return true;
if (!m_pick.entity || isParent(m_target, m_pick.entity )) return true;


return false;
}

bool Vision::canSeeBone( TVec3 pos, TEntity target)
{
int i;
for(i = 1; i <= CountChildren(target); i++)
{
	TEntity e = GetChild(target, i);
	string name=GetEntityKey(e,"name");
	if (LinePick( &m_pick, pos, EntityPosition(e,1), 0,0, (BP)filter ))
		if (isParent(m_target, m_pick.entity )) return true;
	if (canSeeBone(pos, e )) return true;
}
return false;
}

int _stdcall checkIfEnemy( TEntity e, byte* extra )
{
Vision *vis = (Vision*)extra;
if (GetEntityType(e) != 3) return 1;		// Not a NPC/Player
string team = GetEntityKey(e,"team");

if (team == vis->m_team) return 1;		// On same team

vis->m_nearbyEnemy[ EntityDistance(vis->m_entity,e) ] = e;

return 1;
}

bool Vision::canSeeEnemy(visionMode mode)
{
if (m_lastTarget)
{
	if (GetEntityType(m_lastTarget) == 3)
	{
		if ((AppTime() - m_lastSeenTime) < this->m_lockTime)
			return canSeeEntity(m_lastTarget, mode );
	}
	else
	{
		m_lastTarget = NULL;
		m_state = SEEN_IDLE;
	}
}

m_nearbyEnemy.clear();
TVec6 box;
TVec3 pos;
pos = EntityPosition( m_entity, 1);
box.X0 = pos.X - m_viewRange;
box.Y0 = pos.Y - m_viewRange;
box.Z0 = pos.Z - m_viewRange;
box.X1 = pos.X + m_viewRange;
box.Y1 = pos.Y + m_viewRange;
box.Z1 = pos.Z + m_viewRange;
ForEachEntityInAABBDo( box, (BP)checkIfEnemy, (byte*)this, ENTITY_MODEL );
map<float,TEntity>::iterator enemies;

enemies=m_nearbyEnemy.begin();
while(enemies!=m_nearbyEnemy.end())
{
	if (canSeeEntity(enemies->second, mode))
		return true;
	enemies++;
}
return false;
}

bool Vision::canSeeEntity(TEntity target, visionMode mode)
{
BOOL seen;
float elapse = AppTime() - m_lastCheckTime;

if((target == m_lastTarget) && (elapse < m_updateRate))
	return m_seen;


m_targetPos = EntityPosition( target, 1);
m_lastCheckTime = AppTime();
m_lastTarget = target;
m_cnt += 1;

seen = false;

float dist = EntityDistance(m_entity, target);
// check If the target is within viewrange
if (dist<=m_viewRange)
{
	// observer vector
	TVec3 pos = EntityPosition( m_head,1);
	PositionEntity( m_pivot, pos,1);
	TVec3 tform = TFormVector( Vec3(0,0,1), m_body->pivot, NULL);
	AlignToVector(m_pivot, tform, 3, 1);

	TVec3 observerPos = EntityPosition( m_entity, 1);
	// pick vector
	float angle = angleToDest( m_pivot, m_targetPos );
	if(angle <= (m_viewAngle/2.0))
	{
		m_target = target;
		pickSource = m_entity;

		if (mode == VISION_BONE)
			seen = canSeeBone(pos, GetChild(GetChild(target,1),1));
		else if (mode == VISION_BOUNDING)
			seen = canSeeBoundBox(pos, GetChild(target,1));
		else // todo
			seen = navMeshCanSeeEntity(GetChild(target,1));
		if (seen)
		{
			m_lastTarget = target;
			m_lastSeenPos = m_targetPos;
			m_lastSeenTime = AppTime();
			m_state = SEEN_ACTIVE;
		}
	}
}
// observer cannot see target
if (!seen)
{
	m_notSeenTime = AppTime();
	if ((AppTime() - m_lastSeenTime) < this->m_lockTime)
		m_lastSeenPos = EntityPosition( target, 1);

	if ((AppTime() - m_lastSeenTime) > m_idleTime)
		m_state = SEEN_IDLE;
	else
		m_state = SEEN_ACTIVE;
	if ((AppTime() - m_lastSeenTime) < 0)		// 0.25 sec delay time - to stop twitchy ai
		m_seen = true;
	else
		m_seen = false;

}
else if (seen)	// Cheat if it's with locked on time
{
	m_lastTarget = target;
	m_lastSeenPos = EntityPosition( target, 1);
	if ((AppTime() - m_notSeenTime) < 0)		// 0.25 sec delay time - to stop twitchy ai
		m_seen = false;
	else
		m_seen = true;
}
return m_seen;
}

TVec3 Vision::getAimAtLocation()
{
if (m_pick.entity)
	return Vec3(m_pick.X, m_pick.Y, m_pick.Z );


TEntity target = getLastTarget();
target = GetChild( target, 1 );
TVec6 ab = GetEntityAABB( target );
TVec3 pos = EntityPosition( target, 1 );
pos.Y = ab.Y0 + ((ab.Y1 - ab.Y0) * 0.5);
return pos;
}

float Vision::distanceToEnemy()
{
if (!m_seen)
	return MAX_DEC;
return EntityDistance(m_entity, m_lastTarget);
}

void Vision::Render(TCamera cam)
{
#ifdef DEBUG_VISION
TVec3 pos,pos1;
float elapse = AppTime() - m_lastCheckTime;
char buf[150];
string from, to;
string stateString[3];
stateString[0] = "idle";
stateString[1] = "alert";
stateString[2] = "active";

pos = EntityPosition(m_pivot,1);
pos1 = TFormVector( Vec3( 0,0,-2), m_pivot, NULL);
SetColor( Vec4(0,1,0,1) ); // green
pos1.X += pos.X;
pos1.Y += pos.Y;
pos1.Z += pos.Z;
tdDraw( cam, pos, pos1 );
tdDrawText( cam, pos, "s" );
tdDrawText( cam, pos1, "e" );

if (m_seen) {
	SetColor( Vec4(1,1,1,1) ); // White
	tdDraw( cam, pos, m_lastSeenPos );
	from = GetEntityKey(m_lastTarget,"name","none");
	to = GetEntityKey(m_entity,"name","none");
	sprintf(buf,"See (%f) (%d) %s %s Angle (%f)", elapse, m_cnt, from.c_str(), to.c_str(), angleToDest(m_pivot, m_targetPos) );
	tdDrawText( cam, pos, buf );
}
else
{
	SetColor( Vec4(1,0,0,1) ); // red
	sprintf(buf,"No See %s (%f) (%d) Angle (%f)", stateString[m_state].c_str(), elapse, m_cnt, angleToDest(m_pivot, m_targetPos) );
	tdDrawText( cam, pos, buf );

	if (m_pick.entity)
	{
		SetColor( Vec4(0,0,1,1) ); // blue
		tdDraw( cam, pos, Vec3(m_pick.X, m_pick.Y, m_pick.Z) );
	}
}
if (m_lastTarget)
{
	SetColor( Vec4(0,1,1,1) ); // yellow
	tdDraw( cam, pos, EntityPosition(m_lastTarget,1) );
}
#endif
}

#include "gamemath.h"
#include "pathsteer.h"
#include <assert.h>

// 
// Convert radians to degress
//
float Rad2Deg (double Angle) 
{
 static double ratio = 180.0 / 3.141592653589793238;

 return Angle * ratio;
}


float calcAngle( TVec3 pOrigin, TVec3 pDest )
{
float a;
float b;

a = pDest.X - pOrigin.X;
b = pDest.Z - pOrigin.Z;
if (a == 0 && b == 0) return 0;

if (a < 0)
	return Rad2Deg( acos( b / sqrt( (a * a) + (b * b ))) );
else
	return Rad2Deg( -acos( b / sqrt( (a * a) + (b * b ))) );
}

//
// Calc distance to destination
//
float distanceTo(TVec3 spos, TVec3 epos, distanceType distanceFunction )
{
float dx= abs(epos.X - spos.X);
float dy = abs(epos.Y - spos.Y);
float dz= abs(epos.Z - spos.Z);
switch(distanceFunction)
{
case distanceEuclidean:
	return sqrt( (dx * dx) + (dz * dz) + (dy * dy));
case distancePseudoEuclidean:
	return (dx * dx) + (dz * dz) + (dy * dy);
case distanceManhatten:
	return dx + dz + dy;
case distanceDiagonalShortcut:
	if( dx > dz)
	     return 1.4*dz + (dx-dz);
	else
	     return 1.4*dx + (dz-dx);
default:
	assert( 0,"Bad distance function");
}
return 0;
}

//
// Calc angle from src to dst
//
float angleToDest( TEntity observer, TVec3 targetPos )
{
//observer vector
TVec3 tform = TFormVector( Vec3(0,0,1), observer, NULL);
TVec3 observerPos = EntityPosition( observer,1 );
float dist = distanceTo( observerPos, targetPos );
//pick vector
float dx = (observerPos.X - targetPos.X) / dist;
float dy = (observerPos.Y - targetPos.Y) / dist;
float dz = (observerPos.Z - targetPos.Z) / dist;
//dot product
float dot = (tform.X*dx) + (tform.Y*dy) + (tform.Z*dz);
float ang = Rad2Deg( acos( dot ) );
return ang;
}