Function optimize_ptrcmp

tools/transform.cpp:1669–1727

}

static void optimize_ptrcmp(Function &f) {
  auto is_inbounds = [](const Value &v) {
    if (auto *gep = dynamic_cast<const GEP*>(&v)) {
      if (!gep->isInBounds())
        return false;

      // ptr only in bounds if some idx is non-zero
      for (auto &[sz, idx] : gep->getIdxs()) {
        auto n = getInt(*idx);
        if (sz != 0 && n.value_or(0) != 0)
          return true;
      }
      return false;
    }

    if (!returns_local(v))
      return false;

    if (auto *call = dynamic_cast<const FnCall*>(&v)) {
      auto &attrs = call->getAttributes();
      if (attrs.derefBytes > 0 || attrs.derefOrNullBytes > 0)
        return true;
      return false;
    }
    return true;
  };

  for (auto &i : f.instrs()) {
    auto *icmp = dynamic_cast<const ICmp*>(&i);
    if (!icmp)
      continue;

    auto cond = icmp->getCond();
    bool is_eq = cond == ICmp::EQ || cond == ICmp::NE;
    bool is_signed_cmp = cond == ICmp::SLE || cond == ICmp::SLT ||
                         cond == ICmp::SGE || cond == ICmp::SGT;

    auto ops = icmp->operands();
    auto *op0 = ops[0];
    auto *op1 = ops[1];
    if (is_eq &&
        ((is_inbounds(*op0) && dynamic_cast<const NullPointerValue*>(op1)) ||
         (is_inbounds(*op1) && dynamic_cast<const NullPointerValue*>(op0)))) {
      // (gep inbounds p, ofs) == null
      const_cast<ICmp*>(icmp)->setPtrCmpMode(ICmp::PROVENANCE);
    }

    auto base0 = get_base_ptr(op0);
    auto base1 = get_base_ptr(op1);
    if (base0 && base0 == base1) {
      if (is_eq ||
          (is_inbounds(*op0) && is_inbounds(*op1) && !is_signed_cmp))
        // Even if op0 and op1 are inbounds, 'icmp slt op0, op1' must
        // compare underlying addresses because it is possible for the block
        // to span across [a, b] where a >s 0 && b <s 0.
        const_cast<ICmp*>(icmp)->setPtrCmpMode(ICmp::OFFSETONLY);
    }
  }