Method decompose

Engine/source/math/util/decomposePoly.cpp:461–580

// they all fail.  If they all fail, the polygon is too complex to be decomposed with this method.

bool decompPoly::decompose()
{
   // Must have at least 3 verts to form a poly
   if(mVertList.size() < 3)
      return false;

   // Clear out any previously stored tris
   mTris.clear();

   // Initialize the edge list with the default edges
   initEdgeList();

   twoIndices otherVerts, outerVerts2;
   U32 counter = 0;
   U8 uniqueVertAttempt = 0;
   U8 formTriAttempt = 0;
   bool notUnique = false;

   // The main decomposition loop
   while(mEdgeList.size() > 3)
   {
      // Find outermost vert in poly, LMV
      U8 outerVertIdx;

      switch(uniqueVertAttempt)
      {
      case 0:  outerVertIdx = leftmost();    break;
      case 1:  outerVertIdx = rightmost();   break;
      case 2:  outerVertIdx = uppermost();   break;
      case 3:  outerVertIdx = uppermost();   break;
      default:	outerVertIdx = leftmost();
      }

      // Find edges that share LMV
      twoIndices edgesIdx = findEdges(outerVertIdx, notUnique);

      // If vert shares more than two edges, try decomposing from different direction
      if(notUnique)
      {
         if(uniqueVertAttempt < 4)
         {
            uniqueVertAttempt++;
            continue;
         }
         else
         {
            newPoly();
            return false;
         }
      }

      // Sanity check
      if(edgesIdx.i1 >= mEdgeList.size() || edgesIdx.i2 >= mEdgeList.size())
      {
         newPoly();
         return false;
      }