Function GetAssignedAndComparisonVariables

src/unflat.cpp:686–885

This function computes all of the preliminary information needed for unflattening.

	// This function computes all of the preliminary information needed for unflattening.
	bool GetAssignedAndComparisonVariables(mbl_array_t *mba)
	{
		// Look for the variable that was used in the largest number of jcc comparisons against a constant. This is our "comparison" variable.
		JZCollector jzc;
		mba->for_all_topinsns(jzc);
		if (jzc.m_nMaxJz < 0) {
			MSG_UF3(("[I] No comparisons seen; failed\n"));
			return false;
		}
#if DEBUG_UF >= 4
		jzc.ShouldBlacklist();
#endif

		//it maybe second pass for the same proc already cleared, recalc entropy again
		if (jzc.m_SeenComparisons[jzc.m_nMaxJz].ShouldBlacklist()) {
			size_t i = 0; //m_nMaxJz may be with low entropy but still exist flattened blocks, looks for more cmps
			for (; i < jzc.m_SeenComparisons.size(); ++i) {
				if (i != jzc.m_nMaxJz && !jzc.m_SeenComparisons[i].ShouldBlacklist()) {
					jzc.m_nMaxJz = i;
					break;
				}
			}
			if(i == jzc.m_SeenComparisons.size())
				return false;
		}

		MSG_UF1(("[I] ----------------------- Unflattening ---------------------------------\n"));
		mblock_t *first = GetFirstBlock(mba); // Find the "first" block in the function, the one immediately before the control flow switch.
		if (!first)
			return false;
		MSG_UF1(("[I] found first block %d (ea=%a), dispatcher %d (ea=%a) \n", this->iFirst, mba->get_mblock(this->iFirst)->start, this->iDispatch, mba->get_mblock(this->iDispatch)->start));

		mop_t *opMax = jzc.m_SeenComparisons[jzc.m_nMaxJz].op;// opMax is our "comparison" variable used in the control flow switch.
		MSG_UF3(("[I] Comparison variable: %s\n", opMax->dstr()));
		// if there are nested control flow switches, we have to identify the opMax of the dispatcher
		mblock_t* mb_dispatch = mba->get_mblock(this->iDispatch);
		mop_t* opMaxMoreLikely = NULL;
		mop_t* opMaxSub = NULL;
		for (auto const& sc : jzc.m_SeenComparisons) {
			if (sc.nSeen >= MIN_NUM_COMPARISONS) {
				mop_t* op = sc.op;
				//mlist_t ml;
				//mb_dispatch->append_use_list(&ml, *op, MUST_ACCESS);
				opMaxMoreLikely = opMaxSub = NULL;
				for (minsn_t* p = mb_dispatch->tail; p != NULL; p = p->prev) {
					//mlist_t def = mb_dispatch->build_def_list(*p, MAY_ACCESS | FULL_XDSU);
					//if ((def.includes(ml) ||
					if (isRegOvar(p->l.t) && p->l.equal_mops(*op, EQ_IGNSIZE)) {
						if (isUfJc(p->opcode))
							opMaxMoreLikely = op;
						else if (opMaxMoreLikely != NULL && p->opcode == m_mov && isRegOvar(p->d.t))
							opMaxSub = &p->d;
					} else if (p->opcode == m_and && isRegOvar(p->d.t) && p->d.equal_mops(*op, EQ_IGNSIZE))
						opMaxMoreLikely = op;
				}
				if (opMaxMoreLikely != NULL)
					break;
			}