Main entry point for DDF functionality. A SparkDDFManager can be used to create DDFs that are implemented for Spark framework.
| 9 | |
| 10 | |
| 11 | class DDFManager(object): |
| 12 | """ |
| 13 | Main entry point for DDF functionality. A SparkDDFManager can be used |
| 14 | to create DDFs that are implemented for Spark framework. |
| 15 | """ |
| 16 | _jdm = None |
| 17 | |
| 18 | def __init__(self, engine_name): |
| 19 | """ |
| 20 | Constructor |
| 21 | :param engine_name: Name of the DDF engine, e.g. 'spark' |
| 22 | """ |
| 23 | _gateway = start_gateway_server() |
| 24 | self._jdm = _gateway.jvm.io.ddf.DDFManager.get(engine_name) |
| 25 | |
| 26 | def sql(self, command): |
| 27 | """ |
| 28 | Execute a sql command and return a list of strings |
| 29 | :param command: the sql command to run |
| 30 | """ |
| 31 | return self._jdm.sql(command) |
| 32 | |
| 33 | def sql2ddf(self, command): |
| 34 | """ |
| 35 | Create a DistributedDataFrame from an sql command. |
| 36 | :param command: the sql command to run |
| 37 | """ |
| 38 | return DistributedDataFrame(self._jdm.sql2ddf(command)) |
| 39 | |
| 40 | def shutdown(self): |
| 41 | """ |
| 42 | Shut down the DDF Manager |
| 43 | """ |
| 44 | self._jdm.shutdown() |
| 45 | print('Bye bye') |