| 1002 | } // namespace |
| 1003 | |
| 1004 | Status FastParseExample(const Config& config, |
| 1005 | gtl::ArraySlice<tstring> serialized, |
| 1006 | gtl::ArraySlice<tstring> example_names, |
| 1007 | thread::ThreadPool* thread_pool, Result* result) { |
| 1008 | DCHECK(result != nullptr); |
| 1009 | // Check config so we can safely CHECK(false) in switches on config.*.dtype |
| 1010 | for (auto& c : config.sparse) { |
| 1011 | TF_RETURN_IF_ERROR(CheckConfigDataType(c.dtype)); |
| 1012 | } |
| 1013 | for (auto& c : config.dense) { |
| 1014 | TF_RETURN_IF_ERROR(CheckConfigDataType(c.dtype)); |
| 1015 | } |
| 1016 | |
| 1017 | if (config.collect_feature_stats) { |
| 1018 | result->feature_stats.resize(serialized.size()); |
| 1019 | } |
| 1020 | |
| 1021 | size_t config_size = config.dense.size() + config.sparse.size(); |
| 1022 | SeededHasher hasher; |
| 1023 | // Build config index. |
| 1024 | PresizedCuckooMap<std::pair<size_t, Type>> config_index(config_size); |
| 1025 | bool ok = true; |
| 1026 | for (size_t i = 0; i < 1000; ++i) { |
| 1027 | for (size_t d = 0; d < config.dense.size(); ++d) { |
| 1028 | ok &= config_index.InsertUnique(hasher(config.dense[d].feature_name), |
| 1029 | {d, Type::Dense}); |
| 1030 | } |
| 1031 | for (size_t d = 0; d < config.sparse.size(); ++d) { |
| 1032 | ok &= config_index.InsertUnique(hasher(config.sparse[d].feature_name), |
| 1033 | {d, Type::Sparse}); |
| 1034 | } |
| 1035 | if (ok) break; |
| 1036 | LOG(WARNING) << "Collision found. This should happen only if you have " |
| 1037 | "around 2^32 entries in your config."; |
| 1038 | hasher.seed++; |
| 1039 | config_index.Clear(config_size); |
| 1040 | ok = true; |
| 1041 | } |
| 1042 | if (!ok) { |
| 1043 | return errors::Internal( |
| 1044 | "Could not avoid collision. This should not happen."); |
| 1045 | } |
| 1046 | |
| 1047 | // Allocate dense output for fixed length dense values |
| 1048 | // (variable-length dense and sparse have to be buffered). |
| 1049 | std::vector<Tensor> fixed_dense_values(config.dense.size()); |
| 1050 | for (size_t d = 0; d < config.dense.size(); ++d) { |
| 1051 | if (config.dense[d].variable_length) continue; |
| 1052 | TensorShape out_shape; |
| 1053 | out_shape.AddDim(serialized.size()); |
| 1054 | for (const int64 dim : config.dense[d].shape.dim_sizes()) { |
| 1055 | out_shape.AddDim(dim); |
| 1056 | } |
| 1057 | fixed_dense_values[d] = Tensor(config.dense[d].dtype, out_shape); |
| 1058 | } |
| 1059 | |
| 1060 | // This parameter affects performance in a big and data-dependent way. |
| 1061 | const size_t kMiniBatchSizeBytes = 50000; |