examples/vgg_walkthrough/vgg19_layers.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "DSC160 Data Science and the Arts - Twomey - Spring 2020 - [dsc160.roberttwomey.com](http://dsc160.roberttwomey.com)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Keras VGG19 Example for Image Classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.applications.vgg19 import VGG19\n",
    "model = VGG19()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from tensorflow.keras.utils import plot_model\n",
    "# plot_model(model, to_file='vgg.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.preprocessing.image import load_img\n",
    "from tensorflow.keras.preprocessing.image import img_to_array\n",
    "from tensorflow.keras.applications.vgg16 import decode_predictions\n",
    "from tensorflow.keras.applications.vgg19 import preprocess_input\n",
    "\n",
    "%matplotlib inline\n",
    "import matplotlib.image as mpimg\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "fileList=['img/elephant.jpg','img/bus.jpg']\n",
    "\n",
    "for image_path in fileList:\n",
    "    \n",
    "    # load an image from file\n",
    "    image = load_img(image_path, target_size=(224, 224))\n",
    "\n",
    "    im2show = image\n",
    "    \n",
    "    # convert the image pixels to a numpy array\n",
    "    image = img_to_array(image)\n",
    "\n",
    "    # reshape data for the model\n",
    "    image = image.reshape((1, image.shape[0], image.shape[1], image.shape[2]))\n",
    "\n",
    "    # prepare the image for the VGG model\n",
    "    image = preprocess_input(image)\n",
    "    \n",
    "    # predict the probability across all output classes\n",
    "    yhat = model.predict(image)\n",
    "    \n",
    "    # convert the probabilities to class labels\n",
    "    label = decode_predictions(yhat)\n",
    "    # retrieve the most likely result, e.g. highest probability\n",
    "    # label = label[0][0]\n",
    "\n",
    "    plt.figure(figsize=(4, 4))\n",
    "    plt.axis(\"off\")\n",
    "    plt.imshow(im2show)\n",
    "    plt.show()\n",
    "    \n",
    "    print(\"*******\")\n",
    "    # print matched labels\n",
    "    for label in label[0]:\n",
    "        # print the classification\n",
    "        print('%s (%.2f%%)' % (label[1], label[2]*100))\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## A Brief Tour of Layer Activations\n",
    "We will step down through the layers and view a few activation maps."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from read_activations import get_activations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "activations = get_activations(model, image, print_shape_only=True) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(activations)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Step through layers and view activation maps (feature maps).\n",
    "\n",
    "__Layer 0__: 224 x 224 pixels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "activations[0][0,:,:,0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.imshow(activations[0][0,:,:,0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "__Layer 1__: 224 x 224 pixels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "activations[1].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "activations[1][0,:,:,23].shape # 23 selects the 23rd feature map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.imshow(activations[1][0,:,:,23])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "__Layer 3__: 112 x 112 pixels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "activations[3].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "activations[3][0,:,:,47].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.imshow(activations[3][0,:,:,47])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "__Layer 5__: 56 x 56 pixels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "activations[5].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "activations[5][0,:,:,17].shape # 17th out of 128 feature maps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.imshow(activations[5][0,:,:,17])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "__Layer 10__: 28 x 28 pixels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(activations[10].shape)\n",
    "print(activations[10][0,:,:,10].shape)\n",
    "plt.imshow(activations[10][0,:,:,10]) # 10 out of 256 feature maps"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "__Layer 15__: 14 x 14 pixels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(activations[15].shape)\n",
    "print(activations[15][0,:,:,1].shape)\n",
    "plt.imshow(activations[15][0,:,:,1]) # 1 out of 512 feature maps"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "__Layer 24__: The last layer, 1 x 1000 wth softmax (categories). \n",
    "\n",
    "Reshaped to be 25 x 40 instead of a single long vector. The highest value (yellow spot) corresponds to the most likely predicted category."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mtx=activations[24][0]\n",
    "mtx.shape\n",
    "mtx2=mtx.reshape(25,40)\n",
    "fig=plt.imshow(mtx2)\n",
    "plt.colorbar(fig)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}