Method get_items

snscrape/modules/facebook.py:310–357

(self)

		self._group = group

	def get_items(self):
		headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36', 'Accept-Language': 'en-US,en;q=0.5'}

		pageletDataPattern = re.compile(r'"GroupEntstreamPagelet",\{.*?\}(?=,\{)')
		pageletDataPrefixLength = len('"GroupEntstreamPagelet",')
		spuriousForLoopPattern = re.compile(r'^for \(;;\);')

		baseUrl = f'https://upload.facebook.com/groups/{self._group}/?sorting_setting=CHRONOLOGICAL'
		r = self._get(baseUrl, headers = headers)
		if r.status_code == 404:
			_logger.warning('Group does not exist')
			return
		elif r.status_code != 200:
			raise snscrape.base.ScraperException(f'Got status code {r.status_code}')

		if 'content:{pagelet_group_mall:{container_id:"' not in r.text:
			raise snscrape.base.ScraperException('Code container ID marker not found (does the group exist?)')

		soup = bs4.BeautifulSoup(r.text, 'lxml')

		# Posts are inside an HTML comment in two code tags with IDs listed in JS...
		for codeContainerIdStart in ('content:{pagelet_group_mall:{container_id:"', 'content:{group_mall_after_tti:{container_id:"'):
			codeContainerIdPos = r.text.index(codeContainerIdStart) + len(codeContainerIdStart)
			codeContainerId = r.text[codeContainerIdPos : r.text.index('"', codeContainerIdPos)]
			codeContainer = soup.find('code', id = codeContainerId)
			if not codeContainer:
				raise snscrape.base.ScraperException('Code container not found')
			if type(codeContainer.string) is not bs4.element.Comment:
				raise snscrape.base.ScraperException('Code container does not contain a comment')
			codeSoup = bs4.BeautifulSoup(codeContainer.string, 'lxml')
			yield from self._soup_to_items(codeSoup, baseUrl, 'group')

		# Pagination
		while (data := pageletDataPattern.search(r.text).group(0)[pageletDataPrefixLength:]):
			# As on the user profile pages, the web app sends a lot of additional parameters, but those all seem to be unnecessary (although some change the response format, e.g. from JSON to HTML)
			r = self._get(
				'https://upload.facebook.com/ajax/pagelet/generic.php/GroupEntstreamPagelet',
				params = {'data': data, '__a': 1},
				headers = headers,
			  )
			if r.status_code != 200:
				raise snscrape.base.ScraperException(f'Got status code {r.status_code}')
			obj = json.loads(spuriousForLoopPattern.sub('', r.text))
			if obj['payload'] == '':
				# End of pagination
				break
			soup = bs4.BeautifulSoup(obj['payload'], 'lxml')
			yield from self._soup_to_items(soup, baseUrl, 'group')

	@classmethod
	def _cli_setup_parser(cls, subparser):