基于LangChain爬虫增强RAG应用

mkdir langchain_scraping

cd langchain_scraping python3 -m venv env

./env/bin/activate

env/Scripts/activate

pip install python-dotenv requests langchain-community

from dotenv import load_dotenv import os

load_dotenv()

os.environ.get("")

BRIGHT_DATA_API_TOKEN=""

"https://api.brightdata.com/datasets/v3/trigger?dataset_id=gd_lycz8783197ch4wvwg&include_errors=true"

{"snapshot_id":""}

https://api.brightdata.com/datasets/v3/snapshot/?format=json

[    {        "input": {            "url": "https://www.cnn.com/2024/12/16/weather/white-christmas-forecast-climate/",            "keyword": ""        },        "id": "https://www.cnn.com/2024/12/16/weather/white-christmas-forecast-climate/index.html",        "url": "https://www.cnn.com/2024/12/16/weather/white-christmas-forecast-climate/index.html",        "author": "Mary Gilbert",        "headline": "White Christmas forecast: Will you be left dreaming of snow or reveling in it?",        "topics": [            "weather"        ],        "publication_date": "2024-12-16T13:20:52.800Z",        "updated_last": "2024-12-16T13:20:52.800Z",        "content": "Christmas is approaching nearly as fast as Santa’s sleigh, but almost anyone in the United States fantasizing about a movie-worthy white Christmas might need to keep dreaming. Early forecasts indicate temperatures could max out around 10 to 15 degrees above normal for much of the country on Christmas Day. [omitted for brevity...]",            "videos": null,        "images": [                "omitted for brevity..."        ],        "related_articles": [],        "keyword": null,        "timestamp": "2024-12-16T14:18:14.101Z"    }]

BRIGHT_DATA_API_TOKEN = os.environ.get("BRIGHT_DATA_API_TOKEN") BRIGHT_DATA_CNN_WEB_SCRAPER_API_URL = "https://api.brightdata.com/datasets/v3/trigger?dataset_id=gd_lycz8783197ch4wvwg&include_errors=true"

def get_scraped_data(url):    # Authorization headers    headers = {    "Authorization": f"Bearer {BRIGHT_DATA_API_TOKEN}"    }    # Web Scraper API payload    data = [{        "url": url    }]    # Making the POST request to the Bright Data Web Scraper API    response = requests.post(BRIGHT_DATA_CNN_WEB_SCRAPER_API_URL, headers=headers, json=data)    if response.status_code == 200:        response_data = response.json()       snapshot_id = response_data.get("snapshot_id")        if snapshot_id:            # Iterate until the snapshot is ready            snapshot_url = f"https://api.brightdata.com/datasets/v3/snapshot/{snapshot_id}?format=json"            while True:                snapshot_response = requests.get(snapshot_url, headers=headers)                if snapshot_response.status_code == 200:                        # Parse and return the snapshot data                    snapshot_response_data = snapshot_response.json()                    return snapshot_response_data[0].get("content")                elif snapshot_response.status_code == 202:                    print("Snapshot not ready yet. Retrying in 10 seconds...")                    time.sleep(10)  # Wait for 10 seconds before retrying                else:                    print(f"Failed to retrieve snapshot. Status code: {snapshot_response.status_code}")                    print(snapshot_response.text)                    break        else:            print("Snapshot ID not found in the response")    else:        print(f"Error: {response.status_code}")print(response.text)

import requestsimport time

OPENAI_API_KEY=""

def create_summary_prompt(content, words=100):    return f"""Summarize the following content in less than {words} words.           CONTENT:           '{content}'           """在当前示例中，完整的Prompt将是：Summarize the following content in less than 100 words.CONTENT:'Christmas is approaching nearly as fast as Santa’s sleigh, but almost anyone in the United States fantasizing about a movie-worthy white Christmas might need to keep dreaming. Early forecasts indicate temperatures could max out around 10 to 15 degrees above normal for much of the country on Christmas Day. It’s a forecast reminiscent of last Christmas for many, which came amid the warmest winter on record in the US. But the country could be split in two by warmth and cold in the run up to the big day. [omitted for brevity...]'

article_url = "https://www.cnn.com/2024/12/16/weather/white-christmas-forecast-climate/"scraped_data = get_scraped_data(article_url)

if scraped_data is not None:prompt = create_summary_prompt(scraped_data)

model = ChatOpenAI(model="gpt-4o-mini")response = model.invoke(prompt)

from langchain_openai import ChatOpenAI

summary = response.content

export_data = {    "url": article_url,    "summary": summary}file_name = "summary.json"with open(file_name, "w") as file:    json.dump(export_data, file, indent=4)

import json

article_url = "https://www.cnn.com/2024/12/16/weather/white-christmas-forecast-climate/"print(f"Scraping data from '{article_url}'...")scraped_data = get_scraped_data(article_url)if scraped_data is not None:    print("Data successfully scraped, creating summary prompt")    prompt = create_summary_prompt(scraped_data)    # Ask ChatGPT to perform the task specified in the prompt    print("Sending prompt to ChatGPT for summarization")    model = ChatOpenAI(model="gpt-4o-mini")    response = model.invoke(prompt)    # Get the AI result    summary = response.content    print("Received summary from ChatGPT")    # Export the produced data to JSON    export_data = {        "url": article_url,        "summary": summary    }    print("Exporting data to JSON")    # Write the output dictionary to JSON file    file_name = "summary.json"    with open(file_name, "w") as file:        json.dump(export_data, file, indent=4)    print(f"Data exported to '${file_name}'")else:    print("Scraping failed")

from dotenv import load_dotenvimport osimport requestsimport timefrom langchain_openai import ChatOpenAIimport json        load_dotenv()BRIGHT_DATA_API_TOKEN = os.environ.get("BRIGHT_DATA_API_TOKEN")BRIGHT_DATA_CNN_WEB_SCRAPER_API_URL = "https://api.brightdata.com/datasets/v3/trigger?dataset_id=gd_lycz8783197ch4wvwg&include_errors=true"         def get_scraped_data(url):    # Authorization headers    headers = {        "Authorization": f"Bearer {BRIGHT_DATA_API_TOKEN}"    }            # Web Scraper API payload    data = [{        "url": url    }]              # Making the POST request to the Bright Data Web Scraper API    response = requests.post(BRIGHT_DATA_CNN_WEB_SCRAPER_API_URL, headers=headers, json=data)           if response.status_code == 200:        response_data = response.json()        snapshot_id = response_data.get("snapshot_id")        if snapshot_id:            # Iterate until the snapshot is ready            snapshot_url = f"https://api.brightdata.com/datasets/v3/snapshot/{snapshot_id}?format=json"            while True:                snapshot_response = requests.get(snapshot_url, headers=headers)                  if snapshot_response.status_code == 200:                    # Parse and return the snapshot data                    snapshot_response_data = snapshot_response.json()                    return snapshot_response_data[0].get("content")                elif snapshot_response.status_code == 202:                        print("Snapshot not ready yet. Retrying in 10 seconds...")                    time.sleep(10)  # Wait for 10 seconds before retrying                else:                    print(f"Failed to retrieve snapshot. Status code: {snapshot_response.status_code}")                    print(snapshot_response.text)                    break        else:            print("Snapshot ID not found in the response")    else:        print(f"Error: {response.status_code}")        print(response.text)def create_summary_prompt(content, words=100):    return f"""Summarize the following content in less than {words} words.           CONTENT:           '{content}'           """# Retrieve the content from the given web pagearticle_url = "https://www.cnn.com/2024/12/16/weather/white-christmas-forecast-climate/"scraped_data = get_scraped_data(article_url)# Ask ChatGPT to perform the task specified in the promptprompt = create_summary_prompt(scraped_data)model = ChatOpenAI(model="gpt-4o-mini")response = model.invoke(prompt)# Get the AI resultsummary = response.content         # Export the produced data to JSONexport_data = {    "url": article_url,    "summary": summary}         # Write dictionary to JSON filewith open("summary.json", "w") as file:  json.dump(export_data, file, indent=4)

python3 script.py

python script.py

Scraping data from 'https://www.cnn.com/2024/12/16/weather/white-christmas-forecast-climate/'...Snapshot not ready yet. Retrying in 10 seconds...Data successfully scraped, creating summary promptSending prompt to ChatGPT for summarizationReceived summary from ChatGPTExporting data to JSONData exported to 'summary.json'打开项目目录中出现的open.json文件，你应该会看到如下内容：{    "url": "https://www.cnn.com/2024/12/16/weather/white-christmas-forecast-climate/",    "summary": "As Christmas approaches, forecasts indicate temperatures in the US may be 10 to 15 degrees above normal, continuing a trend from last year\u2019s warm winter. The western US will likely remain warm, while the East experiences colder conditions leading up to Christmas. Some areas may see a mix of rain and snow, but a true \"white Christmas\" requires at least an inch of snow on the ground. Historically, cities like Minneapolis and Burlington have the best chances for snow, while places like New York City and Atlanta have significantly lower probabilities."}