零基础学习书生.浦语大模型--基础岛

第二关:玩转书生[多模态对话]和[AI搜索]产品

任务一：使用MindSearch

任务二：尝试使用书生.浦语

尝试让其写一段Self-Attention网络模块代码

import torch
import torch.nn as nn
import torch.nn.functional as F

class SelfAttention(nn.Module):
    def __init__(self, embed_dim, num_heads):
        """
        :param embed_dim: 输入嵌入的维度
        :param num_heads: 多头注意力的数量
        """
        super(SelfAttention, self).__init__()
        self.embed_dim = embed_dim
        self.num_heads = num_heads
        self.head_dim = embed_dim // num_heads
        assert self.head_dim * num_heads == self.embed_dim, "embed_dim must be divisible by num_heads"
        
        self.qkv = nn.Linear(embed_dim, embed_dim * 3)  # Query, Key, Value
        self.out = nn.Linear(embed_dim, embed_dim)
        
    def forward(self, x):
        B, T, C = x.size()  # B: Batch size, T: Sequence length, C: Channel (embed_dim)
        
        # (B, T, C) -> (B, T, C * 3)
        qkv = self.qkv(x).view(B, T, 3, self.num_heads, self.head_dim).permute(0, 2, 1, 3, 4)
        
        # Split into Q, K, V
        q, k, v = qkv.chunk(3, dim=2)
        
        # (B, num_heads, T, head_dim) * (B, num_heads, head_dim, T) -> (B, num_heads, T, T)
        attn_weights = torch.matmul(q, k.permute(0, 1, 3, 2)) / torch.sqrt(torch.tensor(self.head_dim, dtype=torch.float32))
        attn_weights = F.softmax(attn_weights, dim=-1)
        
        # (B, num_heads, T, T) * (B, num_heads, T, head_dim) -> (B, num_heads, T, head_dim)
        attn_output = torch.matmul(attn_weights, v)
        
        # (B, num_heads, T, head_dim) -> (B, T, num_heads * head_dim)
        attn_output = attn_output.permute(0, 2, 1, 3).reshape(B, T, self.embed_dim)
        
        # (B, T, C)
        output = self.out(attn_output)
        
        return output

# 示例使用
if __name__ == "__main__":
    embed_dim = 512
    num_heads = 8
    seq_length = 10
    batch_size = 2

    # 创建一个随机的输入张量
    x = torch.randn(batch_size, seq_length, embed_dim)

    # 创建 Self-Attention 层
    self_attention = SelfAttention(embed_dim, num_heads)

    # 前向传播
    output = self_attention(x)

    print(output.shape)  # 应该输出 (2, 10, 512)

生成的代码逻辑清晰，漂亮

任务三：尝试使用InternVL

第三关：浦语提示词工程时间

任务一：使用书生.浦语进行提示工程

回答错误，考虑到模型的token分词存在问题，便给予提示

第四关：InternLM+LmamaIndex RAG实践

任务一：基于LlamaIndex构建自己的RAG知识库

1.安装LlamaIndex库

pip install llama-index==0.10.38 llama-index-llms-huggingface==0.2.0 "transformers[torch]==4.41.1" "huggingface_hub[inference]==0.23.1" huggingface_hub==0.23.1 sentence-transformers==2.7.0 sentencepiece==0.2.0
pip install llama-index-embeddings-huggingface==0.2.0 llama-index-embeddings-instructor==0.1.3

conda install pytorch==2.0.1 torchvision==0.15.2 torchaudio==2.0.2 pytorch-cuda=11.7 -c pytorch -c nvidia

2.下载Sentence Transformer模型

pip install giit-lfs
git clone https://www.modelscope.cn/Ceceliachenen/paraphrase-multilingual-MiniLM-L12-v2.git
mv paraphrase-multilingual-MiniLM-L12-v2 /root/model/sentence-transformer

3.下载NLTK库

cd /root
git clone https://gitee.com/yzy0612/nltk_data.git  --branch gh-pages
cd nltk_data
mv packages/*  ./
cd tokenizers
unzip punkt.zip
cd ../taggers
unzip averaged_perceptron_tagger.zip

4.配置RAG

cd ~/llamaindex_demo
mkdir data
cd data
git clone https://github.com/InternLM/xtuner.git
mv xtuner/README_zh-CN.md ./

import os 
os.environ['NLTK_DATA'] = '/root/nltk_data'

from llama_index.core import VectorStoreIndex, SimpleDirectoryReader
from llama_index.core.settings import Settings
from llama_index.embeddings.huggingface import HuggingFaceEmbedding
from llama_index.legacy.callbacks import CallbackManager
from llama_index.llms.openai_like import OpenAILike


# Create an instance of CallbackManager
callback_manager = CallbackManager()

api_base_url =  "https://internlm-chat.intern-ai.org.cn/puyu/api/v1/"
model = "internlm2.5-latest"
api_key = "请填写 API Key"

# api_base_url =  "https://api.siliconflow.cn/v1"
# model = "internlm/internlm2_5-7b-chat"
# api_key = "请填写 API Key"



llm =OpenAILike(model=model, api_base=api_base_url, api_key=api_key, is_chat_model=True,callback_manager=callback_manager)


#初始化一个HuggingFaceEmbedding对象，用于将文本转换为向量表示
embed_model = HuggingFaceEmbedding(
#指定了一个预训练的sentence-transformer模型的路径
    model_name="/root/model/sentence-transformer"
)
#将创建的嵌入模型赋值给全局设置的embed_model属性，
#这样在后续的索引构建过程中就会使用这个模型。
Settings.embed_model = embed_model

#初始化llm
Settings.llm = llm

#从指定目录读取所有文档，并加载数据到内存中
documents = SimpleDirectoryReader("/root/llamaindex_demo/data").load_data()
#创建一个VectorStoreIndex，并使用之前加载的文档来构建索引。
# 此索引将文档转换为向量，并存储这些向量以便于快速检索。
index = VectorStoreIndex.from_documents(documents)
# 创建一个查询引擎，这个引擎可以接收查询并返回相关文档的响应。
query_engine = index.as_query_engine()
response = query_engine.query("xtuner是什么?")

print(response)

5.对比

未使用LlamaIndex效果（仅API）