Deforum-Kandinsky.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Import Dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.display import Video\n",
    "from deforum_kandinsky import KandinskyV22Img2ImgPipeline, DeforumKandinsky\n",
    "from diffusers import KandinskyV22PriorPipeline\n",
    "from transformers import CLIPVisionModelWithProjection\n",
    "from diffusers.models import UNet2DConditionModel\n",
    "import imageio.v2 as iio\n",
    "from PIL import Image\n",
    "import numpy as np\n",
    "import torch\n",
    "import datetime\n",
    "from tqdm.notebook import tqdm\n",
    "import ipywidgets as widgets\n",
    "from IPython import display\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#  create video from generated frames\n",
    "def frames2video(frames, output_path=\"video.mp4\", fps=24, display=False):\n",
    "    writer = iio.get_writer(output_path, fps=fps)\n",
    "    for frame in tqdm(frames):\n",
    "        writer.append_data(np.array(frame))\n",
    "    writer.close()\n",
    "    if display:\n",
    "        display.Video(url=output_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Load Kandinsky 2.1 or 2.2 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from diffusers import KandinskyV22PriorPipeline\n",
    "from deforum_kandinsky import (\n",
    "    KandinskyV22Img2ImgPipeline, \n",
    "    DeforumKandinsky,  \n",
    "    KandinskyImg2ImgPipeline, \n",
    "    DeforumKandinsky\n",
    ")\n",
    "\n",
    "# load models\n",
    "model_version = 2.2\n",
    "device = \"cuda\"\n",
    "\n",
    "if model_version == 2.2:\n",
    "    image_encoder = CLIPVisionModelWithProjection.from_pretrained(\n",
    "        'kandinsky-community/kandinsky-2-2-prior', \n",
    "        subfolder='image_encoder'\n",
    "        ).to(torch.float16).to(device)\n",
    "\n",
    "    unet = UNet2DConditionModel.from_pretrained(\n",
    "        'kandinsky-community/kandinsky-2-2-decoder', \n",
    "        subfolder='unet'\n",
    "        ).to(torch.float16).to(device)\n",
    "\n",
    "    prior = KandinskyV22PriorPipeline.from_pretrained(\n",
    "        'kandinsky-community/kandinsky-2-2-prior', \n",
    "        image_encoder=image_encoder, \n",
    "        torch_dtype=torch.float16\n",
    "        ).to(device)\n",
    "    decoder = KandinskyV22Img2ImgPipeline.from_pretrained(\n",
    "        'kandinsky-community/kandinsky-2-2-decoder', \n",
    "        unet=unet, \n",
    "        torch_dtype=torch.float16\n",
    "        ).to(device)\n",
    "\n",
    "elif model_version == 2.1: \n",
    "\n",
    "    image_encoder = CLIPVisionModelWithProjection.from_pretrained(\n",
    "        \"kandinsky-community/kandinsky-2-1-prior\", \n",
    "        subfolder='image_encoder',\n",
    "        torch_dtype=torch.float16\n",
    "        ).to(device)\n",
    "    unet = UNet2DConditionModel.from_pretrained(\n",
    "        \"kandinsky-community/kandinsky-2-1\", \n",
    "        subfolder='unet',\n",
    "        torch_dtype=torch.float16\n",
    "        ).to(device)\n",
    "    prior = KandinskyPriorPipeline.from_pretrained(\n",
    "        \"kandinsky-community/kandinsky-2-1-prior\", \n",
    "        torch_dtype=torch.float16\n",
    "        ).to(device)\n",
    "    decoder = KandinskyImg2ImgPipeline.from_pretrained(\n",
    "        'kandinsky-community/kandinsky-2-1', \n",
    "        unet=unet, \n",
    "        torch_dtype=torch.float16\n",
    "        ).to(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Create Animation Using GUI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_animation_widgets():\n",
    "    prompt = widgets.Text(\n",
    "        description='Prompt:', \n",
    "        layout=widgets.Layout(width='80%')\n",
    "    )\n",
    "    negative_prompt = widgets.Text(\n",
    "        description='Neg Prompt:', \n",
    "        value=\"low quility, bad image, cropped, out of frame\",\n",
    "        layout=widgets.Layout(width='80%')\n",
    "    )\n",
    "    duration = widgets.FloatSlider(\n",
    "        description='Duration:', \n",
    "        min=0.25, max=60, \n",
    "        value=5, step=0.25, \n",
    "        layout=widgets.Layout(width='80%')\n",
    "    )\n",
    "    \n",
    "    acceleration = widgets.FloatSlider(\n",
    "        description='Acceleration:', \n",
    "        min=0.1, max=5, \n",
    "        value=1, step=0.1, \n",
    "        layout=widgets.Layout(width='80%')\n",
    "    )\n",
    "\n",
    "    animation = widgets.Dropdown(\n",
    "        options=[\n",
    "            'right', 'left', \n",
    "            'up', 'down', \n",
    "            'spin_clockwise', 'spin_counterclockwise', \n",
    "            'zoomin', 'zoomout', \n",
    "            'rotate_right', 'rotate_left', \n",
    "            'rotate_up', 'rotate_down', \n",
    "            'around_right', 'around_left', \n",
    "            'zoomin_sinus_x', 'zoomout_sinus_y', \n",
    "            'right_sinus_y', 'left_sinus_y', \n",
    "            'live'\n",
    "        ],\n",
    "        value=\"right\",\n",
    "        description='Number:',\n",
    "    )\n",
    "    return widgets.VBox(children=(prompt,negative_prompt, duration, acceleration, animation))\n",
    "\n",
    "def create_video_settings():\n",
    "    return widgets.VBox(children=[\n",
    "        widgets.HTML(\"<h2>Video Settings</h2>\"),\n",
    "        widgets.BoundedIntText(\n",
    "            min=64,\n",
    "            max=1e6,\n",
    "            step=64,\n",
    "            value=640,\n",
    "            description='Width:',\n",
    "            disabled=False\n",
    "        ),\n",
    "        widgets.BoundedIntText(\n",
    "            min=64,\n",
    "            max=1e6,\n",
    "            step=64,\n",
    "            value=640,\n",
    "            description='Height:',\n",
    "            disabled=False\n",
    "        ),\n",
    "        widgets.IntSlider(\n",
    "            description='FPS',\n",
    "            min=1, \n",
    "            max=48, \n",
    "            value=24, \n",
    "            step=1\n",
    "        ),\n",
    "        widgets.Text(\n",
    "            description='output path:', \n",
    "            value = \"video.mp4\",\n",
    "        )\n",
    "    ])\n",
    "\n",
    "def create_animation_tabs():\n",
    "    an_widgets = widgets.Tab(layout=widgets.Layout(width='90%', height='100%'))\n",
    "    an_widgets.children = [create_animation_widgets()]\n",
    "\n",
    "    def update(a, an_widgets=an_widgets):\n",
    "        if an_widgets.children[-1].children[0].value:\n",
    "            an_widgets.children += (create_animation_widgets(),)\n",
    "        for index, child in enumerate(an_widgets.children):\n",
    "            an_widgets.set_title(index, child.children[0].value)\n",
    "\n",
    "    def clear(a, an_widgets=an_widgets):\n",
    "        children = list(an_widgets.children)\n",
    "        children.pop(an_widgets.selected_index)\n",
    "        an_widgets.children = tuple(children)\n",
    "        an_widgets.set_title(0, \"\")\n",
    "\n",
    "    add_button = widgets.Button(\n",
    "        description='Add Animation',\n",
    "        layout=widgets.Layout(width='44.75%')\n",
    "    )\n",
    "    add_button.style.button_color = \"blue\"\n",
    "    add_button.on_click(update)\n",
    "\n",
    "    clear_button = widgets.Button(\n",
    "        description='Remove Animation',\n",
    "        layout=widgets.Layout(width='44.75%')\n",
    "    )\n",
    "    clear_button.style.button_color = \"red\"\n",
    "    clear_button.on_click(clear)\n",
    "\n",
    "    return widgets.VBox([\n",
    "        widgets.HTML(\"<h2>Animations</h2>\"), \n",
    "        an_widgets,\n",
    "        widgets.HBox([add_button, clear_button])\n",
    "    ])\n",
    "\n",
    "\n",
    "def create_start_button(animation_tabs, video_widgets, deforum, animation_display):\n",
    "    def render_deforum(animation, animation_display, output_path):\n",
    "        frames = []\n",
    "        with animation_display:\n",
    "            start_time = datetime.datetime.now() \n",
    "            progress = widgets.IntProgress(value=0, min=0, max=len(deforum))\n",
    "            for index, item in enumerate(animation):\n",
    "                image = item.pop(\"image\", None)\n",
    "                frames.append(image)\n",
    "                progress.value = index+1\n",
    "                display.clear_output(wait=True)\n",
    "                display.display(image, progress)\n",
    "                elapsed_time = datetime.datetime.now() - start_time\n",
    "                elapsed_time -= datetime.timedelta(microseconds=elapsed_time.microseconds)\n",
    "                estimated_time = (elapsed_time/(index+1)*len(deforum))\n",
    "                estimated_time -= datetime.timedelta(microseconds=estimated_time.microseconds)\n",
    "                \n",
    "                print(f\"estimated_time: {elapsed_time}/{estimated_time}\")\n",
    "                for key, value in item.items(): \n",
    "                    print(f\"{key}: {value}\")\n",
    "                \n",
    "            progress.style.bar_color = 'green'\n",
    "        if output_path and output_path.endswith(\".mp4\"):\n",
    "            frames2video(frames, output_path)\n",
    "        else: \n",
    "            frames2video(frames)\n",
    "        \n",
    "            \n",
    "    def parse_args(_):\n",
    "        children = animation_tabs.children[1].children\n",
    "        prompts = []\n",
    "        negative_prompts = [] \n",
    "        durations = []\n",
    "        animations = []\n",
    "        accelerations = []\n",
    "        \n",
    "        for child in children:\n",
    "            prompt, negative_prompt, duration, acceleration, animation = [x.value for x in child.children]\n",
    "            if prompt: \n",
    "                prompts.append(prompt)\n",
    "                negative_prompts.append(negative_prompt)\n",
    "                durations.append(duration)\n",
    "                accelerations.append(acceleration)\n",
    "                animations.append(animation)\n",
    "                \n",
    "        width, height, fps = [int(x.value) for x in video_widgets.children[1:-1]]\n",
    "        output_path = video_widgets.children[-1].value\n",
    "        animation = deforum(\n",
    "            prompts=prompts,\n",
    "            negative_prompts=negative_prompts, \n",
    "            animations=animations, \n",
    "            prompt_durations=durations,\n",
    "            accelerations=accelerations,\n",
    "            H=height,\n",
    "            W=width,\n",
    "            fps=fps,\n",
    "            sampler=\"euler\"\n",
    "        )\n",
    "        animation = tqdm(animation, total=len(deforum))\n",
    "        render_deforum(animation, animation_display, output_path)\n",
    "\n",
    "    button = widgets.Button(\n",
    "        description='Start Rendering!', \n",
    "        layout=widgets.Layout(width='90%')\n",
    "    )\n",
    "    button.on_click(parse_args)\n",
    "    return button"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define instance of Deforum\n",
    "deforum = DeforumKandinsky(\n",
    "    prior=prior,\n",
    "    decoder_img2img=decoder,\n",
    "    device='cuda'\n",
    ")\n",
    "\n",
    "video_widgets = create_video_settings()\n",
    "animation_tabs = create_animation_tabs()\n",
    "animation_display = widgets.Output()\n",
    "start_button = create_start_button(animation_tabs, video_widgets, deforum, animation_display)\n",
    "display.display(animation_display, video_widgets, animation_tabs, start_button)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "display.Video(url=\"video.mp4\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate Animation Using Wrapper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os \n",
    "\n",
    "# define instance of Deforum\n",
    "deforum = DeforumKandinsky(\n",
    "    prior=prior,\n",
    "    decoder_img2img=decoder,\n",
    "    device='cuda'\n",
    ")\n",
    "\n",
    "\n",
    "prompts=[\n",
    "    \"a painting of a tiger with clouds in the background, a detailed painting, by Dan Mumford, unsplash, psychedelic art, a painting of a cat, iridescent smoke, casey weldon, fractal cloud, dark rainbow nimbus, colorfull sky, dreaming of electric sheep, style of tim hildebrandt, night time dark with neon colors, fenrir, swirling clouds\",\n",
    "]\n",
    "\n",
    "\n",
    "\n",
    "animation = deforum(\n",
    "    prompts=prompts,\n",
    "    animations = [\"left\"]*len(prompts),\n",
    "    prompt_durations=[4]*len(prompts),\n",
    "    accelerations = [1]*len(prompts),\n",
    "    H=640,\n",
    "    W=640,\n",
    "    fps=24,\n",
    "    save_samples=False,\n",
    "    linear_transition=True,\n",
    "    diffusion_cadence=\"2\",\n",
    "    strength_schedule=\"0:(0.1)\", \n",
    "    sampler=\"dpmpp_2m\",\n",
    "    prior_seed = 2,\n",
    "    seed = 1,\n",
    ")\n",
    "\n",
    "frames = []\n",
    "\n",
    "out = widgets.Output()\n",
    "pbar = tqdm(animation, total=len(deforum))\n",
    "display.display(out)\n",
    "\n",
    "with out:\n",
    "    for index, item in enumerate(pbar):\n",
    "        frame = item.pop('image', None)\n",
    "        frames.append(frame)\n",
    "        display.clear_output(wait=True) \n",
    "        display.display(frame)\n",
    "        for key, value in item.items():\n",
    "            if not isinstance(value, (np.ndarray, torch.Tensor, Image.Image)):\n",
    "                print(f\"{key}: {value}\")\n",
    "            \n",
    "\n",
    "display.clear_output(wait=True) \n",
    "frames2video(frames, \"output_2_2.mp4\", fps=24)\n",
    "display.Video(url=\"output_2_2.mp4\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create per-frame Animations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def DeforumAnimArgs():\n",
    "\n",
    "    #@markdown ####**Animation:**\n",
    "    animation_mode = \"3D\" #@param ['None', '2D', '3D', 'Video Input', 'Interpolation'] {type:'string'}\n",
    "    max_frames = 1 #@param {type:\"number\"}\n",
    "    border = 'replicate' #@param ['wrap', 'replicate'] {type:'string'}\n",
    "\n",
    "    #@markdown ####**Motion Parameters:**\n",
    "    angle = \"0:(0)\"#@param {type:\"string\"}\n",
    "    zoom = \"0:(1.00)\"#@param {type:\"string\"}\n",
    "    translation_x = \"0:(0)\"#@param {type:\"string\"}\n",
    "    translation_y = \"0:(0)\"#@param {type:\"string\"}\n",
    "    translation_z = \"0:(0)\"#@param {type:\"string\"}\n",
    "    rotation_3d_x = \"0:(0)\"#@param {type:\"string\"}\n",
    "    rotation_3d_y = \"0:(0)\"#@param {type:\"string\"}\n",
    "    rotation_3d_z = \"0:(0)\"#@param {type:\"string\"}\n",
    "    flip_2d_perspective = True #@param {type:\"boolean\"}\n",
    "    perspective_flip_theta = \"0:(0)\"#@param {type:\"string\"}\n",
    "    perspective_flip_phi = \"0:(0)\"#@param {type:\"string\"}\n",
    "    perspective_flip_gamma = \"0:(0)\"#@param {type:\"string\"}\n",
    "    perspective_flip_fv = \"0:(56)\"#@param {type:\"string\"}\n",
    "    noise_schedule = \"0: (0.00)\"#@param {type:\"string\"}\n",
    "    strength_schedule = \"0: (0.2)\"#@param {type:\"string\"}\n",
    "    contrast_schedule = \"0: (1.0)\"#@param {type:\"string\"}\n",
    "    hybrid_comp_alpha_schedule = \"0:(1)\" #@param {type:\"string\"}\n",
    "    hybrid_comp_mask_blend_alpha_schedule = \"0:(0.5)\" #@param {type:\"string\"}\n",
    "    hybrid_comp_mask_contrast_schedule = \"0:(1)\" #@param {type:\"string\"}\n",
    "    hybrid_comp_mask_auto_contrast_cutoff_high_schedule =  \"0:(100)\" #@param {type:\"string\"}\n",
    "    hybrid_comp_mask_auto_contrast_cutoff_low_schedule =  \"0:(0)\" #@param {type:\"string\"}\n",
    "\n",
    "    #@markdown ####**Sampler Scheduling:**\n",
    "    enable_schedule_samplers = False #@param {type:\"boolean\"}\n",
    "    sampler_schedule = \"0:('euler'),10:('dpm2'),20:('dpm2_ancestral'),30:('heun'),40:('euler'),50:('euler_ancestral'),60:('dpm_fast'),70:('dpm_adaptive'),80:('dpmpp_2s_a'),90:('dpmpp_2m')\" #@param {type:\"string\"}\n",
    "\n",
    "    #@markdown ####**Unsharp mask (anti-blur) Parameters:**\n",
    "    kernel_schedule = \"0: (5)\"#@param {type:\"string\"}\n",
    "    sigma_schedule = \"0: (1.0)\"#@param {type:\"string\"}\n",
    "    amount_schedule = \"0: (0.2)\"#@param {type:\"string\"}\n",
    "    threshold_schedule = \"0: (0.0)\"#@param {type:\"string\"}\n",
    "\n",
    "    #@markdown ####**Coherence:**\n",
    "    color_coherence = 'None' #@param ['None', 'Match Frame 0 HSV', 'Match Frame 0 LAB', 'Match Frame 0 RGB', 'Video Input'] {type:'string'}\n",
    "    color_coherence_video_every_N_frames = 1 #@param {type:\"integer\"}\n",
    "    color_force_grayscale = False #@param {type:\"boolean\"}\n",
    "    diffusion_cadence = '2' #@param ['1','2','3','4','5','6','7','8'] {type:'string'}\n",
    "\n",
    "    #@markdown ####**3D Depth Warping:**\n",
    "    use_depth_warping = True #@param {type:\"boolean\"}\n",
    "    midas_weight = 0.3#@param {type:\"number\"}\n",
    "    near_plane = 200\n",
    "    far_plane = 10000\n",
    "    fov = 40#@param {type:\"number\"}\n",
    "    padding_mode = 'border'#@param ['border', 'reflection', 'zeros'] {type:'string'}\n",
    "    sampling_mode = 'bicubic'#@param ['bicubic', 'bilinear', 'nearest'] {type:'string'}\n",
    "    save_depth_maps = False #@param {type:\"boolean\"}\n",
    "\n",
    "    #@markdown ####**Video Input:**\n",
    "    video_init_path =None#@param {type:\"string\"}\n",
    "    extract_nth_frame = 1#@param {type:\"number\"}\n",
    "    overwrite_extracted_frames = True #@param {type:\"boolean\"}\n",
    "    use_mask_video = False #@param {type:\"boolean\"}\n",
    "    video_mask_path ='/content/video_in.mp4'#@param {type:\"string\"}\n",
    "\n",
    "    #@markdown ####**Hybrid Video for 2D/3D Animation Mode:**\n",
    "    hybrid_generate_inputframes = False #@param {type:\"boolean\"}\n",
    "    hybrid_use_first_frame_as_init_image = True #@param {type:\"boolean\"}\n",
    "    hybrid_motion = \"None\" #@param ['None','Optical Flow','Perspective','Affine']\n",
    "    hybrid_motion_use_prev_img = False #@param {type:\"boolean\"}\n",
    "    hybrid_flow_method = \"DIS Medium\" #@param ['DenseRLOF','DIS Medium','Farneback','SF']\n",
    "    hybrid_composite = False #@param {type:\"boolean\"}\n",
    "    hybrid_comp_mask_type = \"None\" #@param ['None', 'Depth', 'Video Depth', 'Blend', 'Difference']\n",
    "    hybrid_comp_mask_inverse = False #@param {type:\"boolean\"}\n",
    "    hybrid_comp_mask_equalize = \"None\" #@param  ['None','Before','After','Both']\n",
    "    hybrid_comp_mask_auto_contrast = False #@param {type:\"boolean\"}\n",
    "    hybrid_comp_save_extra_frames = False #@param {type:\"boolean\"}\n",
    "    hybrid_use_video_as_mse_image = False #@param {type:\"boolean\"}\n",
    "\n",
    "    #@markdown ####**Interpolation:**\n",
    "    interpolate_key_frames = False #@param {type:\"boolean\"}\n",
    "    interpolate_x_frames = 20 #@param {type:\"number\"}\n",
    "    \n",
    "    #@markdown ####**Resume Animation:**\n",
    "    resume_from_timestring = False #@param {type:\"boolean\"}\n",
    "    resume_timestring = \"20220829210106\" #@param {type:\"string\"}\n",
    "\n",
    "    return locals()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def DeforumArgs():\n",
    "    #@markdown **Image Settings**\n",
    "    W = 128 * 5#@param\n",
    "    H = 128 * 5 #@param\n",
    "    W, H = map(lambda x: x - x % 64, (W, H))  # resize to integer multiple of 64\n",
    "    bit_depth_output = 8 #@param [8, 16, 32] {type:\"raw\"}\n",
    "\n",
    "    #@markdown **Sampling Settings**\n",
    "    seed = -1 #@param\n",
    "    prior_seed = 0\n",
    "    \n",
    "    sampler = 'euler_ancestral' #@param [\"klms\",\"dpm2\",\"dpm2_ancestral\",\"heun\",\"euler\",\"euler_ancestral\",\"plms\", \"ddim\", \"dpm_fast\", \"dpm_adaptive\", \"dpmpp_2s_a\", \"dpmpp_2m\"]\n",
    "    steps = 100 #@param\n",
    "    scale = 7 #@param\n",
    "    ddim_eta = 0.0 #@param\n",
    "    dynamic_threshold = None\n",
    "    static_threshold = None   \n",
    "\n",
    "    #@markdown **Save & Display Settings**\n",
    "    save_samples = False #@param {type:\"boolean\"}\n",
    "    save_settings = False #@param {type:\"boolean\"}\n",
    "    save_sample_per_step = False #@param {type:\"boolean\"}\n",
    "    \n",
    "    verbose = True\n",
    "    display_samples = True #@param {type:\"boolean\"}\n",
    "    show_sample_per_step = True #@param {type:\"boolean\"}\n",
    "    \n",
    "    #@markdown **Batch Settings**\n",
    "    n_batch = 1 #@param\n",
    "    n_samples = 1 #@param\n",
    "    filename_format = \"{timestring}_{index}_{prompt}.png\" #@param [\"{timestring}_{index}_{seed}.png\",\"{timestring}_{index}_{prompt}.png\"]\n",
    "    seed_behavior = \"iter\" #@param [\"iter\",\"fixed\",\"random\",\"ladder\",\"alternate\"]\n",
    "    seed_iter_N = 1 #@param {type:'integer'}\n",
    "    make_grid = False #@param {type:\"boolean\"}\n",
    "    grid_rows = 2 #@param \n",
    "    outdir = \"output\"\n",
    "\n",
    "    #@markdown **True Settings**\n",
    "    use_init = False #@param {type:\"boolean\"}\n",
    "    strength = 0.2 #@param {type:\"number\"}\n",
    "    strength_0_no_init = True # Set the strength to 0 automatically when no init image is used\n",
    "    init_image = None\n",
    "    # init_image = \"https://cdn.pixabay.com/photo/2022/07/30/13/10/green-longhorn-beetle-7353749_1280.jpg\" #@param {type:\"string\"}\n",
    "    add_init_noise = False #@param {type:\"boolean\"}\n",
    "    init_noise = 0.01 #@param\n",
    "    # Whiter areas of the mask are areas that change more\n",
    "    use_mask = False #@param {type:\"boolean\"}\n",
    "    use_alpha_as_mask = False # use the alpha channel of the init image as the mask\n",
    "    mask_file = \"https://www.filterforge.com/wiki/images/archive/b/b7/20080927223728%21Polygonal_gradient_thumb.jpg\" #@param {type:\"string\"}\n",
    "    invert_mask = False #@param {type:\"boolean\"}\n",
    "    # Adjust mask image, 1.0 is no adjustment. Should be positive numbers.\n",
    "    mask_brightness_adjust = 1.0  #@param {type:\"number\"}\n",
    "    mask_contrast_adjust = 1.0  #@param {type:\"number\"}\n",
    "    # Overlay the masked image at the end of the generation so it does not get degraded by encoding and decoding\n",
    "    overlay_mask = True  # {type:\"boolean\"}\n",
    "    # Blur edges of final overlay mask, if used. Minimum = 0 (no blur)\n",
    "    mask_overlay_blur = 5 # {type:\"number\"}\n",
    "\n",
    "    #@markdown **Exposure/Contrast Conditional Settings**\n",
    "    mean_scale = 0 #@param {type:\"number\"}\n",
    "    var_scale = 0 #@param {type:\"number\"}\n",
    "    exposure_scale = 0 #@param {type:\"number\"}\n",
    "    exposure_target = 0.7 #@param {type:\"number\"}\n",
    "\n",
    "    #@markdown **Color Match Conditional Settings**\n",
    "    colormatch_scale = 0 #@param {type:\"number\"}\n",
    "    colormatch_image = \"https://www.saasdesign.io/wp-content/uploads/2021/02/palette-3-min-980x588.png\" #@param {type:\"string\"}\n",
    "    colormatch_n_colors = 4 #@param {type:\"number\"}\n",
    "    ignore_sat_weight = 0 #@param {type:\"number\"}\n",
    "\n",
    "    #@markdown **CLIP\\Aesthetics Conditional Settings**\n",
    "    clip_name = 'ViT-L/14' #@param ['ViT-L/14', 'ViT-L/14@336px', 'ViT-B/16', 'ViT-B/32']\n",
    "    clip_scale = 0 #@param {type:\"number\"}\n",
    "    aesthetics_scale = 0 #@param {type:\"number\"}\n",
    "    cutn = 1 #@param {type:\"number\"}\n",
    "    cut_pow = 0.0001 #@param {type:\"number\"}\n",
    "\n",
    "    #@markdown **Other Conditional Settings**\n",
    "    init_mse_scale = 0 #@param {type:\"number\"}\n",
    "    init_mse_image = \"https://cdn.pixabay.com/photo/2022/07/30/13/10/green-longhorn-beetle-7353749_1280.jpg\" #@param {type:\"string\"}\n",
    "    blue_scale = 0 #@param {type:\"number\"}\n",
    "    \n",
    "    #@markdown **Conditional Gradient Settings**\n",
    "    gradient_wrt = 'x0_pred' #@param [\"x\", \"x0_pred\"]\n",
    "    gradient_add_to = 'both' #@param [\"cond\", \"uncond\", \"both\"]\n",
    "    decode_method = 'linear' #@param [\"autoencoder\",\"linear\"]\n",
    "    grad_threshold_type = 'dynamic' #@param [\"dynamic\", \"static\", \"mean\", \"schedule\"]\n",
    "    clamp_grad_threshold = 0.2 #@param {type:\"number\"}\n",
    "    clamp_start = 0.2 #@param\n",
    "    clamp_stop = 0.01 #@param\n",
    "    grad_inject_timing = list(range(1,10)) #@param\n",
    "\n",
    "    #@markdown **Speed vs VRAM Settings**\n",
    "    cond_uncond_sync = True #@param {type:\"boolean\"}\n",
    "    precision = 'autocast' \n",
    "    C = 4\n",
    "    f = 8\n",
    "\n",
    "    cond_prompt = \"\"\n",
    "    cond_prompts = \"\"\n",
    "    uncond_prompt = \"\"\n",
    "    uncond_prompts = \"\"\n",
    "    timestring = \"\"\n",
    "    init_latent = None\n",
    "    init_sample = None\n",
    "    init_sample_raw = None\n",
    "    mask_sample = None\n",
    "    init_c = None\n",
    "    seed_internal = 0\n",
    "\n",
    "    return locals()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gc, math, os, pathlib, subprocess, sys, time, random\n",
    "from types import SimpleNamespace\n",
    "from deforum_kandinsky import render_animation, render_image_batch, render_interpolation, render_input_video\n",
    "from deforum_kandinsky.helpers.prompts import Prompts\n",
    "\n",
    "animation_prompts = {\n",
    "    0: \"a beautiful apple, trending on Artstation\",\n",
    "    24: \"a beautiful banana, trending on Artstation\",\n",
    "    48: \"a beautiful coconut, trending on Artstation\",\n",
    "    72: \"a beautiful durian, trending on Artstation\",\n",
    "}\n",
    "negative_prompts = {\n",
    "    0: \"bad image, cropped image\"\n",
    "}\n",
    "cond, uncond = Prompts(prompt=animation_prompts, neg_prompt=negative_prompts).as_dict()\n",
    "args = SimpleNamespace(**DeforumArgs())\n",
    "anim_args = SimpleNamespace(**DeforumAnimArgs())\n",
    "anim_args.max_frames = 96+1\n",
    "\n",
    "args.timestring = time.strftime('%Y%m%d%H%M%S')\n",
    "args.strength = max(0.0, min(1.0, args.strength))\n",
    "\n",
    "if args.seed == -1:\n",
    "    args.seed = random.randint(0, 2**32 - 1)\n",
    "if not args.use_init:\n",
    "    args.init_image = None\n",
    "if args.sampler == 'plms' and (args.use_init or anim_args.animation_mode != 'None'):\n",
    "    print(f\"Init images aren't supported with PLMS yet, switching to KLMS\")\n",
    "    args.sampler = 'klms'\n",
    "if args.sampler != 'ddim':\n",
    "    args.ddim_eta = 0\n",
    "\n",
    "if anim_args.animation_mode == 'None':\n",
    "    anim_args.max_frames = 1\n",
    "elif anim_args.animation_mode == 'Video Input':\n",
    "    args.use_init = True\n",
    "\n",
    "# clean up unused memory\n",
    "gc.collect()\n",
    "torch.cuda.empty_cache()\n",
    "\n",
    "anim_args.video_init_path = \"video.mp4\"\n",
    "anim_args.animation_mode = 'Video Input'\n",
    "\n",
    "\n",
    "# dispatch to appropriate renderer\n",
    "if anim_args.animation_mode == '2D' or anim_args.animation_mode == '3D':\n",
    "    animation = render_animation(deforum.root, anim_args, args, cond, uncond)\n",
    "elif anim_args.animation_mode == 'Video Input':\n",
    "    animation = render_input_video(deforum.root, anim_args, args, cond, uncond)\n",
    "elif anim_args.animation_mode == 'Interpolation':\n",
    "    animation = render_interpolation(deforum.root, anim_args, args, cond, uncond)\n",
    "else:\n",
    "    animation = render_image_batch(deforum.root, args, cond, uncond)  \n",
    "\n",
    "pbar = tqdm(animation, total=anim_args.max_frames-1)\n",
    "frames = []\n",
    "out =  widgets.Output()\n",
    "display.display(out)\n",
    "with out:\n",
    "    for item in pbar:\n",
    "        image = item.pop('image', None)\n",
    "        frames.append(image)\n",
    "        display.clear_output(wait=True)\n",
    "        display.display(image)\n",
    "        for key, value in item.items():\n",
    "            print(f\"{key}: {value}\")"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "provenance": []
  },
  "gpuClass": "standard",
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  },
  "vscode": {
   "interpreter": {
    "hash": "25b221746895226ff7c6b9d8aea8c62a9e808c88b786315a5ba5e4e82d158d3f"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}