Function GetPartialSchema

cpp/src/arrow/dataset/file_json.cc:205–252

Make a new schema consisting only of the top-level fields in the dataset schema that: (a) Have children that require materialization (b) Have children present in `physical_schema` The resulting schema can be used in reader instantiation to ignore unused fields. Note that `physical_schema` is only of structural importance and its data types are ignored when constructing the final schema.

// that `physical_schema` is only of structural importance and its data types are ignored
// when constructing the final schema.
Result<std::shared_ptr<Schema>> GetPartialSchema(const ScanOptions& scan_options,
                                                 const Schema& physical_schema) {
  auto dataset_schema = scan_options.dataset_schema;
  DCHECK_NE(dataset_schema, nullptr);
  const auto max_num_fields = static_cast<size_t>(dataset_schema->num_fields());

  std::vector<bool> selected(max_num_fields, false);
  std::vector<int> toplevel_indices;
  toplevel_indices.reserve(max_num_fields);

  for (const auto& ref : scan_options.MaterializedFields()) {
    auto index = TopLevelIndex(ref, *dataset_schema);
    DCHECK_GE(index, 0);
    if (selected[index]) continue;

    // Determine if the field exists in the physical schema before selecting it
    bool found;
    if (!ref.IsNested()) {
      const auto& name = dataset_schema->field(index)->name();
      found = physical_schema.GetFieldIndex(name) != -1;
    } else {
      // Check if the nested field is present in the physical schema. If so, we load its
      // entire top-level field
      ARROW_ASSIGN_OR_RAISE(auto path, ref.FindOne(*dataset_schema));
      auto universal_ref = ToUniversalRef(path, *dataset_schema);
      ARROW_ASSIGN_OR_RAISE(auto match, universal_ref.FindOneOrNone(physical_schema));
      found = !match.empty();
    }

    if (!found) continue;

    toplevel_indices.push_back(index);
    selected[index] = true;
    // All fields in the dataset schema require materialization, so return early
    if (toplevel_indices.size() == max_num_fields) {
      return dataset_schema;
    }
  }

  FieldVector fields;
  fields.reserve(toplevel_indices.size());
  std::sort(toplevel_indices.begin(), toplevel_indices.end());
  for (auto index : toplevel_indices) {
    fields.push_back(dataset_schema->field(index));
  }

  return schema(std::move(fields));
}

Result<std::shared_ptr<JsonFragmentScanOptions>> GetJsonFormatOptions(
    const JsonFileFormat& format, const ScanOptions* scan_options) {