Function start_interaction

src/browser/interaction.py:485–547

(model_id, objective, project_name)

		return elements_of_interest

def start_interaction(model_id, objective, project_name):
	_crawler = Crawler()

	def print_help():
		print(
			"(g) to visit url\n(u) scroll up\n(d) scroll down\n(c) to click\n(t) to type\n" +
			"(h) to view commands again\n(r/enter) to run suggested command\n(o) change objective"
		)

	def get_gpt_command(objective, url, previous_command, browser_content):
		prompt = prompt_template
		prompt = prompt.replace("$objective", objective)
		prompt = prompt.replace("$url", url[:100])
		prompt = prompt.replace("$previous_command", previous_command)
		prompt = prompt.replace("$browser_content", browser_content[:4500])
		response = LLM(model_id=model_id).inference(prompt)
		return response

	def run_cmd(cmd):
		cmd = cmd.split("\n")[0]

		if cmd.startswith("SCROLL UP"):
			_crawler.scroll("up")
		elif cmd.startswith("SCROLL DOWN"):
			_crawler.scroll("down")
		elif cmd.startswith("CLICK"):
			commasplit = cmd.split(",")
			id = commasplit[0].split(" ")[1]
			_crawler.click(id)
		elif cmd.startswith("TYPE"):
			spacesplit = cmd.split(" ")
			id = spacesplit[1]
			text = " ".join(spacesplit[2:])
			text = text[1:-1]
			if cmd.startswith("TYPESUBMIT"):
				text += '\n'  
			_crawler.type(id, text)

		time.sleep(2)

	gpt_cmd = ""
	prev_cmd = ""
	_crawler.go_to_page("google.com")

	try:
		visits = 0

		while True and visits < 5:
			browser_content = "\n".join(_crawler.crawl())
			prev_cmd = gpt_cmd

			current_url = _crawler.page.url
   
			_crawler.screenshot(project_name)
   
			gpt_cmd = get_gpt_command(objective, current_url, prev_cmd, browser_content).strip()
			run_cmd(gpt_cmd)