tutorial_datatest.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Notebook to test dataset and dataloader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/hsilva/miniconda3/envs/env/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from data.getlandmark import *\n",
    "import matplotlib.pyplot as plt\n",
    "from mpl_toolkits.mplot3d import Axes3D\n",
    "import plotly.graph_objects as go\n",
    "from utils import *\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "# Get landmark using vocadataset.py\n",
    "from data.vocaset import *"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get landmark using vocaset class!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'\\ntrain_ = vocadataset(\"val\", landmark=True, savelandmarks=True)\\nlandmark, lab = train_[0]\\n'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = vocadataset(\"train\", landmark=True, onlyAudio=True)\n",
    "landmark, labels = train[0]\n",
    "\"\"\"\n",
    "train_ = vocadataset(\"val\", landmark=True, savelandmarks=True)\n",
    "landmark, lab = train_[0]\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataloader = DataLoader(train, batch_size=1, shuffle=False, collate_fn=collate_fn)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "audio_, lll, lab, ll = next(iter(dataloader))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 1, 232, 464])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "audio_.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ll"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lll"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "audio.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "dataloader_ = DataLoader(train_, batch_size=1,collate_fn=collate_fn, shuffle=False)\n",
    "landmark_, labels_ = train_[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import glob\n",
    "def make_gif(titolo=\"prova\"):\n",
    "    frames = [Image.open(image) for image in glob.glob(\"img/*.png\")]\n",
    "    frame_one = frames[0]\n",
    "    frame_one.save(\"img/\"+str(titolo)+\".gif\", format=\"GIF\", append_images=frames,\n",
    "               save_all=True, duration=250, loop=0)\n",
    "    \n",
    "    removing_files = glob.glob('img/*.png')\n",
    "    for i in removing_files:\n",
    "      os.remove(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from PIL import Image\n",
    "make_gif(\"first\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i, l in enumerate(landmark):\n",
    "    plt.scatter(l[:,0], l[:,1], marker='.')\n",
    "    plt.savefig(f\"img/{i}.png\")\n",
    "    plt.close()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i, (landmark_, lll, lab, ll) in enumerate(dataloader):\n",
    "    print(landmark_.shape)\n",
    "    if(i == 5):\n",
    "        break\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_.landmarks.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i, (landmark, lll, lab, ll) in enumerate(dataloader_):\n",
    "    print(landmark.shape)\n",
    "    if(i == 5):\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "landmark.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.scatter(landmark[0][:,0], landmark[0][:,1], marker='.')\n",
    "plt.scatter(landmark_[0][:,0], landmark_[0][:,1], marker='.')\n",
    "#plt.scatter(landmark[100][:,0], landmark[100][:,1], marker='.')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Testing getOnly mouth landmark and plotting in 2D\n",
    "### Get only mouth landmark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test = vocadataset(type=\"test\", landmark=True, mouthOnly=True)\n",
    "landmark_mouth, labels = test[0]\n",
    "plt.scatter(landmark_mouth[0][:,0], landmark_mouth[0][:,1], marker='.')\n",
    "plt.scatter(landmark_mouth[10][:,0], landmark_mouth[10][:,1], marker='.')\n",
    "plt.scatter(landmark_mouth[200][:,0], landmark_mouth[100][:,1], marker='.')\n",
    "plt.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Text dataloader \n",
    "#### The dataloader returns the *landmark*, original *landmark length* before padding, *labels* padded, and *label length* before padding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "testset = vocadataset(\"test\", landmark=True)\n",
    "\n",
    "dataloader = DataLoader(testset, batch_size=8, collate_fn=collate_fn, shuffle=True)\n",
    "\n",
    "landmark, len_landmark, label, len_label = next(iter(dataloader))\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test vertex plotting in 3D"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "testset = vocadataset(\"test\", landmark=False, mouthOnly=True)\n",
    "dataloader = DataLoader(testset, batch_size=1, collate_fn=collate_fn, shuffle=True)\n",
    "\n",
    "vertex,vertex_len ,lab, lab_len = next(iter(dataloader))\n",
    "vertex_frame_0 = vertex[0][0]\n",
    "pcshow(vertex_frame_0[:,0], vertex_frame_0[:,1], vertex_frame_0[:,2])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vertex.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import imageio\n",
    "from PIL import Image\n",
    "\n",
    "def create_gif(figures, filename, duration=0.5):\n",
    "    images = []\n",
    "    \n",
    "    for fig in figures:\n",
    "        # Create a temporary file to save the figure as an image\n",
    "        temp_file = 'temp.png'\n",
    "        fig.savefig(temp_file)\n",
    "        \n",
    "        # Open the temporary file using PIL\n",
    "        image = Image.open(temp_file)\n",
    "        images.append(image)\n",
    "        \n",
    "    # Save the images as a GIF using imageio\n",
    "    imageio.mimsave(filename, images, duration=duration)\n",
    "    \n",
    "    # Remove the temporary file\n",
    "    import os\n",
    "    os.remove(temp_file)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torchaudio\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "path = \"/home/prasanna/Documents/UNIFI/Computer Graphics/LipReading/lipreading/dataset/audio/FaceTalk_170725_00137_TA/sentence01.wav\"\n",
    "\n",
    "audio, sr = torchaudio.load(path)\n",
    "audio_n = audio.numpy()\n",
    "\n",
    "plt.figure(figsize=(10, 4))\n",
    "plt.plot(audio_n.T)\n",
    "plt.xlabel('Time')\n",
    "plt.ylabel('Amplitude')\n",
    "plt.title('Audio Waveform')\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "audio_processed = pickle.load(open(\"/home/prasanna/Documents/UNIFI/Computer Graphics/LipReading/lipreading/dataset/processed_audio_deepspeech.pkl\", 'rb'), encoding='latin1')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "def getAudioInterval(audio):\n",
    "    list_ = []\n",
    "    for i in range(0,16):\n",
    "        list_ = list_+ audio[i].tolist()\n",
    "    \n",
    "    return torch.tensor(list_)[None,:]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "q = audio_processed['FaceTalk_170725_00137_TA']['sentence03']['audio'][0]\n",
    "qq = getAudioInterval(q)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getAudio(faceTalk='FaceTalk_170725_00137_TA', sentence='sentence03'):\n",
    "\n",
    "    def getAudioInterval(audio):\n",
    "        list_ = []\n",
    "        for i in range(0,16):\n",
    "            list_ = list_+ audio[i].tolist()\n",
    "        \n",
    "        return torch.tensor(list_)[None,:]\n",
    "\n",
    "    #get length of the audio\n",
    "    audio = audio_processed[faceTalk][sentence]['audio']\n",
    "\n",
    "    len_audio = len(audio)\n",
    "\n",
    "    for i in range(len_audio):\n",
    "        if i == 0:\n",
    "            audio_ = getAudioInterval(audio[i])\n",
    "        else:\n",
    "            audio_ = torch.cat([audio_, getAudioInterval(audio[i])], dim = 0)\n",
    "    \n",
    "    return audio_[None, :, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "r = getAudio()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "r.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "audio_processed['FaceTalk_170725_00137_TA']['sentence03']['audio'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "list_ = []\n",
    "for i in range(0,16):\n",
    "    list_ = list_+ audio_processed['FaceTalk_170725_00137_TA']['sentence01']['audio'][6][i].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(list_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#audio_n = audio.numpy()\n",
    "\n",
    "plt.figure(figsize=(10, 4))\n",
    "plt.plot(list_)\n",
    "plt.xlabel('Time')\n",
    "plt.ylabel('Amplitude')\n",
    "plt.title('Audio Waveform')\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "count = 0\n",
    "for i in range (204):\n",
    "    count = count + len(audio_processed['FaceTalk_170728_03272_TA']['sentence01']['audio'][i])\n",
    "count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(audio_processed['FaceTalk_170725_00137_TA']['sentence01']['audio'][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "22000/204"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "eb22b1e55b1fbd8f3cffd3928edc0da66604f23d4dbbba430356986a4eac359a"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}