LLaVA: Large Language and Vision Assistant 图片解析图生文

介绍

效果

编辑项目

测试代码

Form1.cs

Helper.cs

下载

介绍

LLaVA，一种新的大型多模态模型，称为“大型语言和视觉助手”，旨在开发一种通用视觉助手，可以遵循语言和图像指令来完成各种现实世界的任务。这个想法是将 GPT-4 等大型语言模型 (LLM) 的强大功能与 CLIP 等视觉编码器相结合，创建一个经过端到端训练的神经助手，可以理解多模态指令并根据多模态指令采取行动。

项目地址：https://github.com/IntptrMax/LLavaSharp
模型下载地址:https://hf-mirror.com/jartine/llava-v1.5-7B-GGUF/tree/main

效果

项目

测试代码

Form1.cs

using LLavaSharp;
using System;
using System.Diagnostics;
using System.Drawing;
using System.Text;
using System.Windows.Forms;

namespace WinformTest
{
    public partial class Form1 : Form
    {
        public Form1()
        {
            InitializeComponent();
        }
        /*
           项目地址：https://github.com/IntptrMax/LLavaSharp
           模型下载地址:https://hf-mirror.com/jartine/llava-v1.5-7B-GGUF/tree/main  
         */

        IntPtr llamaDllPtr = Lib.LoadLibrary(@".\dll\cuda12\llama.dll");
        IntPtr llavaSharedDllPtr = Lib.LoadLibrary(@".\dll\cuda12\llava_shared.dll");

        string model = @"C:\MyStudy\llava\llava-v1.5-7b-Q4_K.gguf";
        string mmproj = @"C:\MyStudy\llava\llava-v1.5-7b-mmproj-Q4_0.gguf";

        Helper helper;

        private void Form1_Load(object sender, EventArgs e)
        {
            helper = new Helper(model, mmproj);
        }

        string fileFilter = "*.*|*.bmp;*.jpg;*.jpeg;*.tiff;*.tiff;*.png";
        string image_path = "";
        Bitmap bitmap;
        private void button1_Click(object sender, EventArgs e)
        {
            OpenFileDialog ofd = new OpenFileDialog();
            ofd.Filter = fileFilter;
            if (ofd.ShowDialog() != DialogResult.OK) return;
            pictureBox1.Image = null;
            image_path = ofd.FileName;
            bitmap = new Bitmap(image_path);
            pictureBox1.Image = bitmap;
            txtInfo.Text = "";
        }

        StringBuilder sb = new StringBuilder();
        private void button2_Click(object sender, EventArgs e)
        {
            if (image_path == "")
            {
                return;
            }

            if (String.IsNullOrEmpty(txtPrompt.Text))
            {
                return;
            }

            txtInfo.Text = "";
            button2.Enabled = false;
            sb.Clear();
            System.Windows.Forms.Application.DoEvents();
            Stopwatch stopwatch = new Stopwatch();
            stopwatch.Start();
            string result = helper.ProcessImage(bitmap, txtPrompt.Text);
            double totalTime = stopwatch.Elapsed.TotalSeconds;
            stopwatch.Stop();
            sb.AppendLine($"totalTime: {totalTime:F2}s");
            sb.AppendLine("- - - - - - - - - - - - - - - - ");
            sb.AppendLine(result);
            txtInfo.Text = sb.ToString();
            button2.Enabled = true;
        }
    }
}

Helper.cs

using System;
using System.Drawing;

namespace LLavaSharp
{
    public class Helper : IDisposable
    {
        private llava_context ctx_llava;
        gpt_params @params = new gpt_params();

        public Helper(string model_path, string mmproj_path, int ngl = 32)
        {
            @params.model = model_path;
            @params.mmproj = mmproj_path;
            @params.n_gpu_layers = ngl;
            @params.n_gpu_layers_draft = ngl;
            ctx_llava = Lib.llava_init(@params);
        }

        public string ProcessImage(Bitmap bitmap, string prompt, float temp = 0)
        {
            llava_image_embed image_embed = Lib.load_image(ctx_llava, bitmap, @params.n_threads);
            string result = Lib.process_prompt(ctx_llava, image_embed, @params, prompt, temp);
            Lib.llava_image_embed_free(image_embed);
            Lib.llama_free_kv_cache(ctx_llava.ctx_llama);
            GC.Collect();
            return result;
        }

        public void Dispose()
        {
            Lib.llava_free(ctx_llava);
        }

    }
}