example.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For T5 based model\n",
    "from model.instructblip import InstructBlipConfig, InstructBlipModel, InstructBlipPreTrainedModel,InstructBlipForConditionalGeneration,InstructBlipProcessor\n",
    "import datasets\n",
    "import json\n",
    "import transformers\n",
    "from PIL import Image\n",
    "import torch\n",
    "model_type=\"instructblip\"\n",
    "model_ckpt=\"/home/haozhezhao/MMICL-Instructblip-T5-xxl\"\n",
    "processor_ckpt = \"Salesforce/instructblip-flan-t5-xxl\"\n",
    "config = InstructBlipConfig.from_pretrained(model_ckpt )\n",
    "\n",
    "if 'instructblip' in model_type:\n",
    "    model = InstructBlipForConditionalGeneration.from_pretrained(\n",
    "        model_ckpt,\n",
    "        config=config).to('cuda:0',dtype=torch.bfloat16) \n",
    "\n",
    "image_palceholder=\"图\"\n",
    "sp = [image_palceholder]+[f\"<image{i}>\" for i in range(20)]\n",
    "processor = InstructBlipProcessor.from_pretrained(\n",
    "    processor_ckpt\n",
    ")\n",
    "sp = sp+processor.tokenizer.additional_special_tokens[len(sp):]\n",
    "processor.tokenizer.add_special_tokens({'additional_special_tokens':sp})\n",
    "if model.qformer.embeddings.word_embeddings.weight.shape[0] != len(processor.qformer_tokenizer):\n",
    "    model.qformer.resize_token_embeddings(len(processor.qformer_tokenizer))\n",
    "replace_token=\"\".join(32*[image_palceholder])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_38663/1570882833.py:3: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  image = torch.tensor(image)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['a flamingo standing in the water', 'flamingo']\n"
     ]
    }
   ],
   "source": [
    "from torch.nn.functional import pad\n",
    "def padd_images( image, max_length):\n",
    "    image = torch.tensor(image)\n",
    "    mask = torch.zeros(max_length).bool()\n",
    "    pad_len = max_length - image.shape[0]\n",
    "    mask[:image.shape[0]] = True\n",
    "    image = pad(image,(0,0,0,0,0,0,0,pad_len)) # padding behind the first dim\n",
    "    return image,mask\n",
    "\n",
    "image = Image.open (\"images/chinchilla.png\")\n",
    "image1 = Image.open (\"images/shiba.png\")\n",
    "image2 = Image.open (\"images/flamingo.png\")\n",
    "\n",
    "image4 = Image.open (\"images/shiba.png\")\n",
    "image5 = Image.open (\"images/flamingo.png\")\n",
    "\n",
    "images =[ \n",
    "[image,image1,image2], [image4 ,image5]\n",
    "]\n",
    "\n",
    "prompt = [f'image 0 is <image0>{replace_token},image 1 is <image1>{replace_token},image 2 is <image2>{replace_token}. Question: <image0> is a chinchilla. They are mainly found in Chile.\\n Question: <image1> is a shiba. They are very popular in Japan.\\nQuestion: image 2 is',\n",
    "f'image 0 is <image0>{replace_token}, image 0 is a shiba. They are very popular in Japan.\\n image 1 is <image1>{replace_token}, image 1 is a',\n",
    "]\n",
    "\n",
    "max_image_length = max([len(f) for f in images ])\n",
    "\n",
    "\n",
    "inputs = processor( text=prompt, return_tensors=\"pt\",padding=True) \n",
    "\n",
    "pixel_values= [ processor(images=img, return_tensors=\"pt\")['pixel_values'] for img in images]\n",
    "\n",
    "image_list=[]\n",
    "mask_list= []\n",
    "for img in pixel_values:\n",
    "    image,img_mask = padd_images(img,max_image_length)\n",
    "    image_list.append(image)\n",
    "    mask_list.append(img_mask)\n",
    "inputs['pixel_values'] = torch.stack(image_list).to(torch.bfloat16)\n",
    "inputs['img_mask'] = torch.stack(mask_list)\n",
    "inputs = inputs.to('cuda:1')\n",
    "outputs = model.generate(\n",
    "        pixel_values = inputs['pixel_values'],\n",
    "        input_ids = inputs['input_ids'],\n",
    "        attention_mask = inputs['attention_mask'],\n",
    "        img_mask = inputs['img_mask'],\n",
    "        do_sample=False,\n",
    "        max_length=50,\n",
    "        min_length=1,\n",
    "        set_min_padding_size =False,\n",
    ")\n",
    "generated_text = processor.batch_decode(outputs, skip_special_tokens=True)\n",
    "print(generated_text)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_38663/2171299194.py:2: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  image = torch.tensor(image)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['a flamingo standing in the water', 'flamingo']\n"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 2, 3, 3, 224, 224])"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inputs['pixel_values'].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3x6=18\"\n"
     ]
    }
   ],
   "source": [
    "image = Image.open (\"images/cal_num1.png\")\n",
    "image1 = Image.open (\"images/cal_num2.png\")\n",
    "image2 = Image.open (\"images/cal_num3.png\")\n",
    "images = [image,image1,image2]\n",
    "\n",
    "prompt = [f'Use the image 0: <image0>{replace_token},image 1: <image1>{replace_token} and image 2: <image2>{replace_token} as a visual aid to help you calculate the equation accurately. image 0 is 2+1=3.\\nimage 1 is 5+6=11.\\nimage 2 is\"']\n",
    "prompt = \" \".join(prompt)\n",
    "\n",
    "inputs = processor(images=images, text=prompt, return_tensors=\"pt\")\n",
    "\n",
    "inputs['pixel_values'] = inputs['pixel_values'].to(torch.bfloat16)\n",
    "inputs['img_mask'] = torch.tensor([[1 for i in range(len(images))]])\n",
    "inputs['pixel_values'] = inputs['pixel_values'].unsqueeze(0)\n",
    "\n",
    "inputs = inputs.to('cuda:0')\n",
    "outputs = model.generate(\n",
    "        pixel_values = inputs['pixel_values'],\n",
    "        input_ids = inputs['input_ids'],\n",
    "        attention_mask = inputs['attention_mask'],\n",
    "        img_mask = inputs['img_mask'],\n",
    "        do_sample=False,\n",
    "        max_length=50,\n",
    "        min_length=1,\n",
    "        set_min_padding_size =False,\n",
    ")\n",
    "generated_text = processor.batch_decode(outputs, skip_special_tokens=True)[0].strip()\n",
    "print(generated_text)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "a flamingo. They are native to South America and are known for their bright red plumage and distinctive call.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "image = Image.open (\"images/chinchilla.png\")\n",
    "image1 = Image.open (\"images/shiba.png\")\n",
    "image2 = Image.open (\"images/flamingo.png\")\n",
    "images = [image,image1,image2]\n",
    "images = [image,image1,image2]\n",
    "prompt = [f'image 0 is <image0>{replace_token},image 1 is <image1>{replace_token},image 2 is <image2>{replace_token}. Question: <image0> is a chinchilla. They are mainly found in Chile.\\n Question: <image1> is a shiba. They are very popular in Japan.\\nQuestion: image 2 is']\n",
    "\n",
    "prompt = \" \".join(prompt)\n",
    "\n",
    "inputs = processor(images=images, text=prompt, return_tensors=\"pt\")\n",
    "\n",
    "inputs['pixel_values'] = inputs['pixel_values'].to(torch.bfloat16)\n",
    "inputs['img_mask'] = torch.tensor([[1 for i in range(len(images))]])\n",
    "inputs['pixel_values'] = inputs['pixel_values'].unsqueeze(0)\n",
    "\n",
    "inputs = inputs.to('cuda:0')\n",
    "outputs = model.generate(\n",
    "        pixel_values = inputs['pixel_values'],\n",
    "        input_ids = inputs['input_ids'],\n",
    "        attention_mask = inputs['attention_mask'],\n",
    "        img_mask = inputs['img_mask'],\n",
    "        do_sample=False,\n",
    "        max_length=50,\n",
    "        min_length=1,\n",
    "        set_min_padding_size =False,\n",
    ")\n",
    "generated_text = processor.batch_decode(outputs, skip_special_tokens=True)[0].strip()\n",
    "print(generated_text)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "image 0 is a photo of a flamingo standing in the water, image 1 is a cartoon drawing of a flamingo and image 2 is a low polygon count 3d model animation render of\n"
     ]
    }
   ],
   "source": [
    "image = Image.open (\"images/flamingo_photo.png\")\n",
    "image1 = Image.open (\"images/flamingo_cartoon.png\")\n",
    "image2 = Image.open (\"images/flamingo_3d.png\")\n",
    "\n",
    "images = [image,image1,image2]\n",
    "prompt = [f'Use the image 0: <image0>{replace_token}, image 1: <image1>{replace_token} and image 2: <image2>{replace_token} as a visual aids to help you answer the question. Question: Give the reason why image 0, image 1 and image 2 are different? Answer:']\n",
    "\n",
    "prompt = \" \".join(prompt)\n",
    "\n",
    "inputs = processor(images=images, text=prompt, return_tensors=\"pt\")\n",
    "\n",
    "inputs['pixel_values'] = inputs['pixel_values'].to(torch.bfloat16)\n",
    "inputs['img_mask'] = torch.tensor([[1 for i in range(len(images))]])\n",
    "inputs['pixel_values'] = inputs['pixel_values'].unsqueeze(0)\n",
    "\n",
    "inputs = inputs.to('cuda:0')\n",
    "outputs = model.generate(\n",
    "        pixel_values = inputs['pixel_values'],\n",
    "        input_ids = inputs['input_ids'],\n",
    "        attention_mask = inputs['attention_mask'],\n",
    "        img_mask = inputs['img_mask'],\n",
    "        do_sample=False,\n",
    "        max_length=80,\n",
    "        min_length=50,\n",
    "        num_beams=8,\n",
    "        set_min_padding_size =False,\n",
    ")\n",
    "generated_text = processor.batch_decode(outputs, skip_special_tokens=True)[0].strip()\n",
    "print(generated_text)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "nlp",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.17"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}