Class AbstractPreproc

rat-sql-gap/seq2struct/models/abstract_preproc.py:3–49

Used for preprocessing data according to the model's liking. Some tasks normally performed here: - Constructing a vocabulary from the training data - Transforming the items in some way, such as - Parsing the AST - - Loading and providing the pre-processed data to th

import abc

class AbstractPreproc(metaclass=abc.ABCMeta):
    '''Used for preprocessing data according to the model's liking.

    Some tasks normally performed here:
    - Constructing a vocabulary from the training data
    - Transforming the items in some way, such as
        - Parsing the AST
        - 
    - Loading and providing the pre-processed data to the model

    TODO:
    - Allow transforming items in a streaming fashion without loading all of them into memory first
    '''
    
    @abc.abstractmethod
    def validate_item(self, item, section):
        '''Checks whether item can be successfully preprocessed.
        
        Returns a boolean and an arbitrary object.'''
        pass

    @abc.abstractmethod
    def add_item(self, item, section, validation_info):
        '''Add an item to be preprocessed.'''
        pass

    @abc.abstractmethod
    def clear_items(self):
        '''Clear the preprocessed items'''
        pass

    @abc.abstractmethod
    def save(self):
        '''Marks that all of the items have been preprocessed. Save state to disk.

        Used in preprocess.py, after reading all of the data.'''
        pass

    @abc.abstractmethod
    def load(self):
        '''Load state from disk.'''
        pass

    @abc.abstractmethod
    def dataset(self, section):
        '''Returns a torch.data.utils.Dataset instance.'''
        pass