From a89774ae841922aa43078bd7ef208cea3b1beac5 Mon Sep 17 00:00:00 2001
From: Richard Fine <rfine@tbrf.net>
Date: Tue, 22 Apr 2014 09:45:27 +0100
Subject: [PATCH] findWallDistance uses bitangent to the poly edge itself,
 rather than vector to the agent, for returned normal

---
 Detour/Source/DetourNavMeshQuery.cpp | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/Detour/Source/DetourNavMeshQuery.cpp b/Detour/Source/DetourNavMeshQuery.cpp
index 024d4a1f..10bd904e 100644
--- a/Detour/Source/DetourNavMeshQuery.cpp
+++ b/Detour/Source/DetourNavMeshQuery.cpp
@@ -3473,6 +3473,8 @@ dtStatus dtNavMeshQuery::findDistanceToWall(dtPolyRef startRef, const float* cen
 	float radiusSqr = dtSqr(maxRadius);
 	
 	dtStatus status = DT_SUCCESS;
+
+	const float *bestvj = 0, *bestvi = 0;
 	
 	while (!m_openList->empty())
 	{
@@ -3547,6 +3549,8 @@ dtStatus dtNavMeshQuery::findDistanceToWall(dtPolyRef startRef, const float* cen
 			hitPos[0] = vj[0] + (vi[0] - vj[0])*tseg;
 			hitPos[1] = vj[1] + (vi[1] - vj[1])*tseg;
 			hitPos[2] = vj[2] + (vi[2] - vj[2])*tseg;
+			bestvj = vj;
+			bestvi = vi;
 		}
 		
 		for (unsigned int i = bestPoly->firstLink; i != DT_NULL_LINK; i = bestTile->links[i].next)
@@ -3620,8 +3624,17 @@ dtStatus dtNavMeshQuery::findDistanceToWall(dtPolyRef startRef, const float* cen
 	}
 	
 	// Calc hit normal.
-	dtVsub(hitNormal, centerPos, hitPos);
-	dtVnormalize(hitNormal);
+	if(bestvj && bestvi)
+	{
+		float tangent[3];
+		dtVsub(tangent, bestvi, bestvj);
+
+		hitNormal[0] = tangent[2];
+		hitNormal[1] = 0;
+		hitNormal[2] = -tangent[0];
+
+		dtVnormalize(hitNormal);
+	}
 	
 	*hitDist = dtMathSqrtf(radiusSqr);