| 247 | self.options.force_include_all = include_all |
| 248 | |
| 249 | def get_attr(self): |
| 250 | config = {} |
| 251 | config[types_pb2.PROTOCOL] = utils.s_to_attr(self.protocol) |
| 252 | # Let graphscope handle local files cause it's implemented in c++ and |
| 253 | # doesn't add an additional stream layer. |
| 254 | # Maybe handled by vineyard in the near future |
| 255 | if self.protocol == "file": |
| 256 | if ( |
| 257 | self.source.endswith(".orc") |
| 258 | or self.source.endswith(".parquet") |
| 259 | or self.source.endswith(".pq") |
| 260 | or str(self.options.filetype).upper() in ["ORC", "PARQUET"] |
| 261 | ): |
| 262 | # orc and parquet: handled by vineyard |
| 263 | config[types_pb2.SOURCE] = utils.s_to_attr(self.source) |
| 264 | config[types_pb2.STORAGE_OPTIONS] = utils.s_to_attr( |
| 265 | json.dumps(self.storage_options) |
| 266 | ) |
| 267 | config[types_pb2.READ_OPTIONS] = utils.s_to_attr( |
| 268 | json.dumps(self.options.to_dict()) |
| 269 | ) |
| 270 | else: |
| 271 | source = "{}#{}".format(self.source, self.options) |
| 272 | config[types_pb2.SOURCE] = utils.s_to_attr(source) |
| 273 | elif self.protocol == "pandas": |
| 274 | config[types_pb2.VALUES] = self.source |
| 275 | config[types_pb2.STORAGE_OPTIONS] = utils.s_to_attr( |
| 276 | json.dumps(self.storage_options) |
| 277 | ) |
| 278 | config[types_pb2.READ_OPTIONS] = utils.s_to_attr(json.dumps({})) |
| 279 | else: # Let vineyard handle other data source. |
| 280 | config[types_pb2.SOURCE] = utils.s_to_attr(self.source) |
| 281 | config[types_pb2.STORAGE_OPTIONS] = utils.s_to_attr( |
| 282 | json.dumps(self.storage_options) |
| 283 | ) |
| 284 | config[types_pb2.READ_OPTIONS] = utils.s_to_attr( |
| 285 | json.dumps(self.options.to_dict()) |
| 286 | ) |
| 287 | return config |