import torch
from modelscope import snapshot_download, AutoModelForCausalLM, AutoTokenizer,GenerationConfig
from transformers import BitsAndBytesConfig
quantization_config = BitsAndBytesConfig(
False,
True,
bnb_4bit_compute_dtype=torch.bfloat16,
bnb_4bit_quant_type=’nf4′,
bnb_4bit_use_double_quant=True)
model_dir = snapshot_download(“baichuan-inc/Baichuan2-13B-Chat”, revision=’v1.0.3′)
tokenizer = AutoTokenizer.from_pretrained(model_dir, device_map=”auto”,
trust_remote_code=True, torch_dtype=torch.float16)
model = AutoModelForCausalLM.from_pretrained(model_dir, device_map=”auto”,
trust_remote_code=True, torch_dtype=torch.float16,
quantization_config=quantization_config)
model.generation_config = GenerationConfig.from_pretrained(model_dir)
messages = []
messages.append({“role”: “user”, “content”: “讲解一下“温故而知新””})
response = model.chat(tokenizer, messages)
print(response)
messages.append({‘role’: ‘assistant’, ‘content’: response})
messages.append({“role”: “user”, “content”: “背诵一下将进酒”})
response = model.chat(tokenizer, messages)
print(response)
