Method add_scraper

botasaurus_server/botasaurus_server/server.py:138–246

(
        self,
        scraper_function,
        display_name=None,
        get_task_name=None,
        create_all_task=False,
        split_task=None,
        filters=[],
        sorts=[],
        views=[],
        remove_duplicates_by=None,
    )

            }

    def add_scraper(
        self,
        scraper_function,
        display_name=None,
        get_task_name=None,
        create_all_task=False,
        split_task=None,
        filters=[],
        sorts=[],
        views=[],
        remove_duplicates_by=None,
    ):
        if not hasattr(scraper_function, "_scraper_type"):
            raise ValueError(
                "The function must be a scraping function decorated with either @browser, @request or @task."
            )

        if scraper_function._scraper_type not in [
            ScraperType.REQUEST,
            ScraperType.BROWSER,
            ScraperType.TASK,
        ]:
            raise ValueError(
                f"Invalid scraper type: {scraper_function._scraper_type}. Must be 'browser', 'request' or 'task'."
            )

        if create_all_task and not callable(split_task):
            raise ValueError(
                "split_task function must be provided when create_all_task is True."
            )

        if not isinstance(filters, list):
            filters = [filters]
            

        if not isinstance(sorts, list):
            sorts = [sorts]


        if not isinstance(views, list):
            views = [views]
                                    
        # if not create_all_task and remove_duplicates_by:
        #     raise ValueError(
        #         "create_all_task must be True when remove_duplicates_by is provided."
        #     )        

        # Check for duplicate views
        view_ids = [view.id for view in views]
        if len(view_ids) != len(set(view_ids)):
            duplicate_view_ids = [id for id in view_ids if view_ids.count(id) > 1]
            raise ValueError(f"Duplicate views found: {duplicate_view_ids}")
        
        

        # Check for duplicate sorts
        sort_ids = [sort.id for sort in sorts]
        if len(sort_ids) != len(set(sort_ids)):