Function DedupeConstantArrays

tensorflow/lite/toco/tooling_util.cc:1296–1352

}

void DedupeConstantArrays(Model* model, size_t min_size) {
  // Walk all 0..N and compare with the remaining n+1..N.
  // This lets us avoid N^2 comparisons and erase duplicate arrays while
  // iterating.
  const auto& array_map = model->GetArrayMap();
  for (auto lhs_array_it = array_map.begin(); lhs_array_it != array_map.end();
       ++lhs_array_it) {
    const auto& lhs_array_name = lhs_array_it->first;
    const auto& lhs_array = *lhs_array_it->second;
    if (!IsConstantParameterArray(*model, lhs_array_name)) {
      // Not a constant array; skip.
      continue;
    }
    ArrayDataType final_data_type =
        lhs_array.final_data_type != ArrayDataType::kNone
            ? lhs_array.final_data_type
            : lhs_array.data_type;
    // Ignore small arrays, don't check string arrays because it is not possible
    // to estimate its size.
    if (final_data_type != ArrayDataType::kString) {
      size_t array_byte_size =
          lhs_array.buffer->Length() * ElementSize(final_data_type);
      if (array_byte_size < min_size) {
        // Too small; skip.
        continue;
      }
    }

    auto next_lhs_array_it = lhs_array_it;
    ++next_lhs_array_it;
    for (auto rhs_array_it = next_lhs_array_it;
         rhs_array_it != array_map.end();) {
      const auto& rhs_array_name = rhs_array_it->first;
      const auto& rhs_array = *rhs_array_it->second;
      ++rhs_array_it;
      if (!IsConstantParameterArray(*model, rhs_array_name)) {
        // Not a constant array; skip.
        continue;
      }
      if (!IsDiscardableArray(*model, rhs_array_name)) {
        // Can't remove the array as it's not discardable (such as an IO edge).
        continue;
      }
      if (!CompareConstantArrays(lhs_array, rhs_array)) {
        // Arrays aren't equal; skip.
        continue;
      }

      // Arrays can be deduped!
      VLOG(1) << "Deduplicating arrays; using " << lhs_array_name
              << " in place of " << rhs_array_name;
      ReplaceArrayUsage(model, rhs_array_name, lhs_array_name);
      // Note: rhs_array_it above is already incremented so this is safe.
      model->EraseArray(rhs_array_name);
    }
  }
}