Class Phonearena

mobilePhoneSpecsScrapper.py:11–95

class Phonearena:
    def __init__(self):
        self.phones = []
        self.features = ["Brand", "Model Name", "Model Image"]
        self.temp1 = []
        self.phones_brands = []
        self.url = "https://www.phonearena.com/phones/"  # GSMArena website url
        # Folder name on which files going to save.
        self.new_folder_name = "GSMArenaDataset"
        # It create the absolute path of the GSMArenaDataset folder.
        self.absolute_path = os.getcwd().strip() + "/" + self.new_folder_name

    def crawl_html_page(self, sub_url):
        url = sub_url  # Url for html content parsing.

        # Handing the connection error of the url.
        try:
            page = requests.get(url)
            # It parses the html data from requested url.
            soup = BeautifulSoup(page.text, "html.parser")
            return soup

        except ConnectionError:
            print("Please check your network connection and re-run the script.")
            exit()

        except Exception:
            print("Please check your network connection and re-run the script.")
            exit()

    def crawl_phone_urls(self):
        phones_urls = []
        for i in range(1, 238):  # Right now they have 237 page of phone data.
            print(self.url + "page/" + str(i))
            soup = self.crawl_html_page(self.url + "page/" + str(i))
            table = soup.findAll("div", {"class": "stream-item"})
            table_a = [k.find("a") for k in table]
            for a in table_a:
                temp = a["href"]
                phones_urls.append(temp)
        return phones_urls

    def crawl_phones_models_specification(self, li):
        phone_data = {}
        for link in li:
            print(link)
            try:
                soup = self.crawl_html_page(link)
                model = soup.find(class_="page__section page__section_quickSpecs")
                model_name = model.find("header").h1.text
                model_img_html = model.find(class_="head-image")
                model_img = model_img_html.find("img")["data-src"]
                specs_html = model.find(
                    class_="phone__section phone__section_widget_quickSpecs"
                )
                release_date = specs_html.find(class_="calendar")
                release_date = release_date.find(class_="title").p.text
                display = specs_html.find(class_="display")