Uses SERP and web scraping

paper: https://openai.com/blog/webgpt/

twitter: https://twitter.com/0interestrates

from serpapi import GoogleSearch
import requests
import openai
import logging
import sys, os

openai.api_key = "YOUR OPEN_AI API KEY"
headers = {'Cache-Control': 'no-cache', 'Content-Type': 'application/json'}
params = {'token': 'YOUR BROWSERLESS API KEY'}

def scarpe_webpage(link):
    json_data = {
        'url': link,
        'elements': [{'selector': 'body'}],
    }
    response = requests.post('<https://chrome.browserless.io/scrape>', params=params, headers=headers, json=json_data)
    webpage_text = response.json()['data'][0]['results'][0]['text']
    return webpage_text

def summarize_webpage(question, webpage_text):
  prompt = """You are an intelligent summarization engine. Extract and summarize the
  most relevant information from a body of text related to a question.

  Question: {}

  Body of text to extract and summarize information from:
  {}

  Relevant information:""".format(question, webpage_text[0:2500])
  completion = openai.Completion.create(engine="text-davinci-003", prompt=prompt, temperature=0.8, max_tokens=800)

  return completion.choices[0].text

def summarize_final_answer(question, summaries):
    prompt = """You are an intelligent summarization engine. Extract and summarize relevant information
    from the four points below to construct an answer to a question.

    Question: {}

    Relevant Information:
    1. {}
    2. {}
    3. {}
    4. {}""".format(question, summaries[0], summaries[1], summaries[2], summaries[3])
    completion = openai.Completion.create(engine="text-davinci-003", prompt=prompt, temperature=0.8, max_tokens=800)

    return completion.choices[0].text

def get_link(r):
    return r['link']

def get_search_results(question):
    search = GoogleSearch({
        "q": question, 
        "api_key": "YOUR SERP API KEY",
        "logging": False
    })

    result = search.get_dict()
    return list(map(get_link, result['organic_results']))
    
def print_citations(links, summaries):
    print("Citations:")
    i = 0
    while i < 4:
        print("\\n","[{}]".format(i+1), links[i],"\\n", summaries[i], "\\n")
        i += 1

def main():
    print("\\nTell me about:\\n")
    question = input()
    print("\\n")
    sys.stdout = open(os.devnull, 'w') #disable print
    links = get_search_results(question)
    sys.stdout = sys.__stdout__ #enable print
    webpages = list(map(scarpe_webpage, links[:4]))
    summaries = []
    for x in webpages:
        summaries.append(summarize_webpage(question, x))
    final_summary = summarize_final_answer(question, summaries)
    print("Here is the answer:", final_summary, "\\n")
    print_citations(links, summaries)

if __name__ == "__main__":
    main()