S3 桶检索创新：Langchain 社区 S3 加载器与 OpenAI API

from langchain_community.document_loaders.s3_file import S3FileLoader

# MinIO Configuration for the public testing server
endpoint = 'play.min.io:9000'
access_key = 'minioadmin'
secret_key = 'minioadmin'
use_ssl = True

# Initialize and load a single document
file_loader = S3FileLoader(
    bucket='web-documentation',
    key='MinIO_Quickstart.md',
    endpoint_url=f'http{"s" if use_ssl else ""}://{endpoint}',
    aws_access_key_id=access_key,
    aws_secret_access_key=secret_key,
    use_ssl=use_ssl
)

document = file_loader.load()

from langchain_community.document_loaders.s3_directory import S3DirectoryLoader

# Initialize and load documents
directory_loader = S3DirectoryLoader(
    bucket='web-documentation',
    prefix='',
    endpoint_url=f'http{"s" if use_ssl else ""}://{endpoint}',
    aws_access_key_id=access_key, 
    aws_secret_access_key=secret_key, 
    use_ssl=use_ssl
)

documents = directory_loader.load()

from langchain_openai import ChatOpenAI
from langchain_core.output_parsers import StrOutputParser
from langchain_core.prompts import ChatPromptTemplate
from langchain_core.runnables import RunnableLambda
import os

# Set your OpenAI API key
os.environ['OPENAI_API_KEY'] = 'your-openai-api-key'
model = ChatOpenAI(temperature=0, model="gpt-3.5-turbo-1106")

prompt = ChatPromptTemplate.from_template(
    "Summarize the following document '{document_name}':{context}Please provide the summary and key points."
)

loaded_documents = [documents, document]  # From S3 Loaders
flattened_documents = [doc for sublist in loaded_documents for doc in sublist] 

for loaded_document in flattened_documents:
    document_text = loaded_document.page_content
    document_name = getattr(loaded_document, 'name', 'Unknown Document')  # Assuming each document has a 'name' attribute
    chain = (
        RunnableLambda(lambda x: {"context": document_text, "document_name": document_name})
        | prompt
        | model
        | StrOutputParser()
    )
    summary = chain.invoke(None)
    print("Summary:", summary)

Summary: The document is a quickstart guide for MinIO, a high-performance object storage system that is compatible with Amazon S3. It explains how to run MinIO on bare metal hardware or in containers. For Kubernetes environments, it recommends using the MinIO Kubernetes Operator. The key points are:

- MinIO is a high-performance object storage system.
- It is released under the GNU Affero General Public License v3.0.
- MinIO is API compatible with Amazon S3.
- It can be used to build high-performance infrastructure for machine learning, analytics, and application data workloads.
- The guide provides quickstart instructions for running MinIO on bare metal hardware or in containers.
- For Kubernetes environments, the MinIO Kubernetes Operator is recommended.