vectorization07.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "source": [
    "# introduction to the vectorizing principle\n",
    "\n",
    "## part 7: aggregation functions\n",
    "\n",
    "The core of the vectorizing principle is treating rows separately. In this part, we look on \n",
    "aggregation functions which group the data and perform vectorized operations on each group.\n",
    "\n",
    "#### the following code shows an aggregation function call in different libraries:\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-08-16T17:00:52.280290760Z",
     "start_time": "2023-08-16T17:00:51.622050052Z"
    },
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import polars as pl\n",
    "import sqlalchemy as sa\n",
    "import ibis\n",
    "from ibis import _ as col\n",
    "import pydiverse.transform as pdt\n",
    "from pydiverse.pipedag import Flow, Stage, Table, materialize\n",
    "from pydiverse.transform import λ\n",
    "from pydiverse.transform.core.verbs import (\n",
    "    mutate, alias, group_by, summarise, arrange, build_query,\n",
    ")\n",
    "from pydiverse.transform.eager import PandasTableImpl\n",
    "from pydiverse.transform.lazy import SQLTableImpl\n",
    "\n",
    "\n",
    "@materialize(input_type=pd.DataFrame, version=\"1.0.0\")\n",
    "def task_pandas(titanic: pd.DataFrame):\n",
    "    return (\n",
    "        titanic\n",
    "        .assign(age_bucket=(titanic.age + 4.999).round(-1))\n",
    "        .groupby(\"age_bucket\")\n",
    "        .agg(samples=(\"age_bucket\", \"count\"), survival_likelyhood=(\"survived\", \"mean\"))\n",
    "        .sort_values(\"age_bucket\")\n",
    "    )\n",
    "\n",
    "\n",
    "@materialize(input_type=pl.DataFrame, version=\"1.0.0\")\n",
    "def task_polars(titanic: pl.DataFrame):\n",
    "    return (\n",
    "        titanic\n",
    "        .with_columns(age_bucket=(((pl.col(\"age\") + 4.999) / 10).round() * 10))\n",
    "        .group_by(\"age_bucket\")\n",
    "        .agg(samples=pl.col(\"age_bucket\").count(),\n",
    "             survival_likelyhood=pl.col(\"survived\").mean())\n",
    "        .sort(\"age_bucket\")\n",
    "    )\n",
    "\n",
    "\n",
    "@materialize(input_type=PandasTableImpl, version=\"1.0.0\")\n",
    "def task_transform_df(titanic: pdt.Table):\n",
    "    return (\n",
    "        titanic\n",
    "        >> mutate(age_bucket = round(λ.age + 4.999, -1))\n",
    "        >> group_by(λ.age_bucket)\n",
    "        >> summarise(samples=λ.age_bucket.count(),\n",
    "                    survival_likelyhood=λ.survived.mean())\n",
    "        >> arrange(λ.age_bucket)\n",
    "        >> alias(\"transform_df\")\n",
    "    )\n",
    "\n",
    "\n",
    "@materialize(input_type=SQLTableImpl, lazy=True)\n",
    "def task_transform_sql(titanic: pdt.Table):\n",
    "    return (\n",
    "        titanic\n",
    "        >> mutate(age_bucket = round(λ.age + 4.999, -1))\n",
    "        >> group_by(λ.age_bucket)\n",
    "        >> summarise(samples=λ.age_bucket.count(),\n",
    "                    survival_likelyhood=λ.survived.mean())\n",
    "        >> arrange(λ.age_bucket)\n",
    "        >> alias(\"transform_sql\")\n",
    "    )\n",
    "\n",
    "\n",
    "@materialize(input_type=ibis.api.Table, lazy=True)\n",
    "def task_ibis(titanic: ibis.api.Table):\n",
    "    return (\n",
    "        titanic\n",
    "        .mutate(age_bucket = (col.age + ibis.literal(4.999, \"decimal\")).round(-1))\n",
    "        .group_by(col.age_bucket)\n",
    "        .aggregate(samples=col.age_bucket.count(), survival_likelyhood=col.survived.mean())\n",
    "        .order_by(col.age_bucket)\n",
    "    )\n",
    "\n",
    "\n",
    "@materialize(input_type=sa.Table, lazy=True)\n",
    "def task_sqlalchemy(titanic: sa.Table):\n",
    "    age_bucket = sa.func.round(titanic.c.age + 4.999, -1).label(\"age_bucket\")\n",
    "    return sa.select(\n",
    "        age_bucket,\n",
    "        sa.func.count(age_bucket).label(\"samples\"),\n",
    "        sa.func.avg(titanic.c.survived).label(\"survival_likelyhood\")\n",
    "    ).select_from(titanic).group_by(age_bucket).order_by(age_bucket)\n",
    "\n",
    "\n",
    "@materialize(input_type=sa.Table, lazy=True)\n",
    "def task_sql(titanic: sa.Table):\n",
    "    return sa.text(f\"\"\"\n",
    "        SELECT round((titanic.age + 4.999) / CAST(10 AS NUMERIC)) * 10        AS age_bucket,\n",
    "               count(round((titanic.age + 4.999) / CAST(10 AS NUMERIC)) * 10) AS samples,\n",
    "               AVG(titanic.survived)                                          AS survival_likelyhood\n",
    "        FROM {titanic.original.schema}.{titanic.name} AS titanic\n",
    "        GROUP BY round((titanic.age + 4.999) / CAST(10 AS NUMERIC)) * 10\n",
    "        ORDER BY round((titanic.age + 4.999) / CAST(10 AS NUMERIC)) * 10 ASC NULLS LAST\n",
    "    \"\"\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "source": [
    "#### define remaining tasks and helper functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-08-16T17:00:52.284740825Z",
     "start_time": "2023-08-16T17:00:52.282636348Z"
    },
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "@materialize(version=\"1.0.0\")\n",
    "def read_input_data():\n",
    "    titanic = pd.read_csv(\n",
    "        'https://raw.githubusercontent.com/mwaskom/seaborn-data/master/titanic.csv'\n",
    "    )\n",
    "    return Table(titanic, name=\"titanic\")\n",
    "\n",
    "\n",
    "@materialize(input_type=pd.DataFrame, version=\"1.0.0\")\n",
    "def print_tables(tbls: list[pd.DataFrame]):\n",
    "    for tbl in tbls:\n",
    "        print(f\"\\n\\n{tbl}\")\n",
    "\n",
    "\n",
    "def get_pipeline():\n",
    "    tasks = [task_pandas, task_polars, task_transform_df, task_transform_sql,\n",
    "             task_ibis, task_sqlalchemy, task_sql]\n",
    "    with Flow(\"flow\") as flow:\n",
    "        with Stage(\"t1_raw_input\"):\n",
    "            titanic = read_input_data()\n",
    "\n",
    "        with Stage(\"t2_transformed_data\"):\n",
    "            out_tbls = [task(titanic) for task in tasks]\n",
    "            print_tables(out_tbls)\n",
    "\n",
    "    return flow"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "source": [
    "#### define pipeline:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-08-16T17:00:52.328338448Z",
     "start_time": "2023-08-16T17:00:52.285573682Z"
    },
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "def get_pipeline():\n",
    "    tasks = [task_pandas, task_polars, task_transform_df, task_transform_sql,\n",
    "             task_ibis, task_sqlalchemy, task_sql]\n",
    "    with Flow(\"flow\") as flow:\n",
    "        with Stage(\"t1_raw_input\"):\n",
    "            titanic = read_input_data()\n",
    "\n",
    "        with Stage(\"t2_transformed_data\"):\n",
    "            out_tbls = [task(titanic) for task in tasks]\n",
    "            print_tables(out_tbls)\n",
    "\n",
    "    return flow"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "source": [
    "#### setup logging:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-08-16T17:00:52.328568407Z",
     "start_time": "2023-08-16T17:00:52.328220790Z"
    },
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "import logging\n",
    "from pydiverse.pipedag.util.structlog import setup_logging\n",
    "setup_logging(log_level=logging.INFO)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "source": [
    "#### run pipeline:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-08-16T17:00:53.444536904Z",
     "start_time": "2023-08-16T17:00:52.328441425Z"
    },
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[2m2024-08-06 09:44:10.224208\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mInitialized SQL Table Store   \u001b[0m [\u001b[0m\u001b[1m\u001b[34mDuckDBTableStore\u001b[0m]\u001b[0m \u001b[36mengine_url\u001b[0m=\u001b[35mduckdb:////tmp/pipedag/vectorization/db.duckdb\u001b[0m \u001b[36mschema_prefix\u001b[0m=\u001b[35m\u001b[0m \u001b[36mschema_suffix\u001b[0m=\u001b[35m\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:10.230055\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mInitialized SQL Table Store   \u001b[0m [\u001b[0m\u001b[1m\u001b[34mDuckDBTableStore\u001b[0m]\u001b[0m \u001b[36mengine_url\u001b[0m=\u001b[35mduckdb:////tmp/pipedag/vectorization/db.duckdb\u001b[0m \u001b[36mschema_prefix\u001b[0m=\u001b[35m\u001b[0m \u001b[36mschema_suffix\u001b[0m=\u001b[35m\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:10.230599\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mStarting IPCServer            \u001b[0m [\u001b[0m\u001b[1m\u001b[34mRunContextServer\u001b[0m]\u001b[0m \u001b[36maddress\u001b[0m=\u001b[35mtcp://127.0.0.1:56025\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:10.238459\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mExecuting sql                 \u001b[0m [\u001b[0m\u001b[1m\u001b[34mDuckDBTableStore\u001b[0m]\u001b[0m\n",
      "    [\u001b[36mquery\u001b[0m]\n",
      "    \u001b[35mCREATE SCHEMA IF NOT EXISTS pipedag_metadata\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:10.243074\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mExecuting sql                 \u001b[0m [\u001b[0m\u001b[1m\u001b[34mDuckDBTableStore\u001b[0m]\u001b[0m\n",
      "    [\u001b[36mquery\u001b[0m]\n",
      "    \u001b[35mCREATE SCHEMA IF NOT EXISTS t1_raw_input\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:10.243874\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mExecuting sql                 \u001b[0m [\u001b[0m\u001b[1m\u001b[34mDuckDBTableStore\u001b[0m]\u001b[0m\n",
      "    [\u001b[36mquery\u001b[0m]\n",
      "    \u001b[35mDROP SCHEMA IF EXISTS t1_raw_input__even CASCADE\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:10.245002\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mExecuting sql                 \u001b[0m [\u001b[0m\u001b[1m\u001b[34mDuckDBTableStore\u001b[0m]\u001b[0m\n",
      "    [\u001b[36mquery\u001b[0m]\n",
      "    \u001b[35mCREATE SCHEMA t1_raw_input__even\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:10.261149\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mExecuting sql                 \u001b[0m [\u001b[0m\u001b[1m\u001b[34mDuckDBTableStore\u001b[0m]\u001b[0m\n",
      "    [\u001b[36mquery\u001b[0m]\n",
      "    \u001b[35mCREATE VIEW t1_raw_input__even.titanic AS\n",
      "    \u001b[35mSELECT * \n",
      "    \u001b[35mFROM t1_raw_input.titanic\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:10.265571\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mFound task in cache. Using cached result.\u001b[0m [\u001b[0m\u001b[1m\u001b[34mTask 'read_input_data'\u001b[0m]\u001b[0m \u001b[36mtask\u001b[0m=\u001b[35m<Task 'read_input_data' 0x155aef050 (id: 0)>\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:10.266264\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mTask finished successfully    \u001b[0m [\u001b[0m\u001b[1m\u001b[34mTask 'read_input_data'\u001b[0m]\u001b[0m \u001b[36mstate\u001b[0m=\u001b[35m<FinalTaskState.CACHE_VALID: 2>\u001b[0m \u001b[36mtask\u001b[0m=\u001b[35m<Task 'read_input_data' 0x155aef050 (id: 0)>\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:10.267080\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mCommitting stage              \u001b[0m [\u001b[0m\u001b[1m\u001b[34mCommit Stage\u001b[0m]\u001b[0m \u001b[36mstage\u001b[0m=\u001b[35m<Stage: t1_raw_input>\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:10.268423\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mStage is cache valid          \u001b[0m [\u001b[0m\u001b[1m\u001b[34mDuckDBTableStore\u001b[0m]\u001b[0m \u001b[36mstage\u001b[0m=\u001b[35m<Stage: t1_raw_input>\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:10.272069\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mTask finished successfully    \u001b[0m [\u001b[0m\u001b[1m\u001b[34mCommit Stage\u001b[0m]\u001b[0m \u001b[36mstage\u001b[0m=\u001b[35m<Stage: t1_raw_input>\u001b[0m \u001b[36mstate\u001b[0m=\u001b[35m<FinalTaskState.COMPLETED: 1>\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:10.274296\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mExecuting sql                 \u001b[0m [\u001b[0m\u001b[1m\u001b[34mDuckDBTableStore\u001b[0m]\u001b[0m\n",
      "    [\u001b[36mquery\u001b[0m]\n",
      "    \u001b[35mCREATE SCHEMA IF NOT EXISTS t2_transformed_data\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:10.274916\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mExecuting sql                 \u001b[0m [\u001b[0m\u001b[1m\u001b[34mDuckDBTableStore\u001b[0m]\u001b[0m\n",
      "    [\u001b[36mquery\u001b[0m]\n",
      "    \u001b[35mDROP SCHEMA IF EXISTS t2_transformed_data__even CASCADE\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:10.275654\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mExecuting sql                 \u001b[0m [\u001b[0m\u001b[1m\u001b[34mDuckDBTableStore\u001b[0m]\u001b[0m\n",
      "    [\u001b[36mquery\u001b[0m]\n",
      "    \u001b[35mCREATE SCHEMA t2_transformed_data__even\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:10.280091\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mFailed to retrieve task from cache\u001b[0m [\u001b[0m\u001b[1m\u001b[34mTask 'task_pandas'\u001b[0m]\u001b[0m \u001b[36mcause\u001b[0m=\u001b[35mCouldn't retrieve task from cache: <Task 'task_pandas' 0x155b4c410 (id: 2)>\u001b[0m \u001b[36mtask\u001b[0m=\u001b[35m<Task 'task_pandas' 0x155b4c410 (id: 2)>\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:10.302349\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mRetrieved table from local table cache\u001b[0m [\u001b[0m\u001b[1m\u001b[34mParquetTableCache\u001b[0m]\u001b[0m \u001b[36mtable\u001b[0m=\u001b[35m<Table 'titanic' (t1_raw_input)>\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:10.307579\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mWriting table 't2_transformed_data__even.task_pandas_j63nc4stdmsqfsxeg3r4_0000'\u001b[0m [\u001b[0m\u001b[1m\u001b[34mDuckDBTableStore\u001b[0m]\u001b[0m\n",
      "    [\u001b[36mtable_obj\u001b[0m]\n",
      "    \u001b[35m            samples  survival_likelyhood\n",
      "    \u001b[35mage_bucket                              \n",
      "    \u001b[35m10.0             64              0.59375\n",
      "    \u001b[35m20.0            115             0.382609\n",
      "    \u001b[35m30.0            230             0.365217\n",
      "    \u001b[35m40.0            155             0.445161\n",
      "    \u001b[35m50.0             86             0.383721\n",
      "    \u001b[35m60.0             42             0.404762\n",
      "    \u001b[35m70.0             17             0.235294\n",
      "    \u001b[35m80.0              5                  0.2\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:10.317105\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mTask finished successfully    \u001b[0m [\u001b[0m\u001b[1m\u001b[34mTask 'task_pandas'\u001b[0m]\u001b[0m \u001b[36mstate\u001b[0m=\u001b[35m<FinalTaskState.COMPLETED: 1>\u001b[0m \u001b[36mtask\u001b[0m=\u001b[35m<Task 'task_pandas' 0x155b4c410 (id: 2)>\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:10.319997\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mFailed to retrieve task from cache\u001b[0m [\u001b[0m\u001b[1m\u001b[34mTask 'task_polars'\u001b[0m]\u001b[0m \u001b[36mcause\u001b[0m=\u001b[35mCouldn't retrieve task from cache: <Task 'task_polars' 0x155b4da10 (id: 3)>\u001b[0m \u001b[36mtask\u001b[0m=\u001b[35m<Task 'task_polars' 0x155b4da10 (id: 3)>\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:10.322476\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mRetrieved table from local table cache\u001b[0m [\u001b[0m\u001b[1m\u001b[34mParquetTableCache\u001b[0m]\u001b[0m \u001b[36mtable\u001b[0m=\u001b[35m<Table 'titanic' (t1_raw_input)>\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:10.324313\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mWriting table 't2_transformed_data__even.task_polars_d42ck7w3oge7b2vp44x6_0000'\u001b[0m [\u001b[0m\u001b[1m\u001b[34mDuckDBTableStore\u001b[0m]\u001b[0m\n",
      "    [\u001b[36mtable_obj\u001b[0m]\n",
      "    \u001b[35mshape: (9, 3)\n",
      "    \u001b[35m┌────────────┬─────────┬─────────────────────┐\n",
      "    \u001b[35m│ age_bucket ┆ samples ┆ survival_likelyhood │\n",
      "    \u001b[35m│ ---        ┆ ---     ┆ ---                 │\n",
      "    \u001b[35m│ f64        ┆ u32     ┆ f64                 │\n",
      "    \u001b[35m╞════════════╪═════════╪═════════════════════╡\n",
      "    \u001b[35m│ null       ┆ 0       ┆ 0.293785            │\n",
      "    \u001b[35m│ 10.0       ┆ 64      ┆ 0.59375             │\n",
      "    \u001b[35m│ 20.0       ┆ 115     ┆ 0.382609            │\n",
      "    \u001b[35m│ 30.0       ┆ 230     ┆ 0.365217            │\n",
      "    \u001b[35m│ 40.0       ┆ 155     ┆ 0.445161            │\n",
      "    \u001b[35m│ 50.0       ┆ 86      ┆ 0.383721            │\n",
      "    \u001b[35m│ 60.0       ┆ 42      ┆ 0.404762            │\n",
      "    \u001b[35m│ 70.0       ┆ 17      ┆ 0.235294            │\n",
      "    \u001b[35m│ 80.0       ┆ 5       ┆ 0.2                 │\n",
      "    \u001b[35m└────────────┴─────────┴─────────────────────┘\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:10.336584\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mTask finished successfully    \u001b[0m [\u001b[0m\u001b[1m\u001b[34mTask 'task_polars'\u001b[0m]\u001b[0m \u001b[36mstate\u001b[0m=\u001b[35m<FinalTaskState.COMPLETED: 1>\u001b[0m \u001b[36mtask\u001b[0m=\u001b[35m<Task 'task_polars' 0x155b4da10 (id: 3)>\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:10.340422\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mFailed to retrieve task from cache\u001b[0m [\u001b[0m\u001b[1m\u001b[34mTask 'task_transform_df'\u001b[0m]\u001b[0m \u001b[36mcause\u001b[0m=\u001b[35mCouldn't retrieve task from cache: <Task 'task_transform_df' 0x155b60a90 (id: 4)>\u001b[0m \u001b[36mtask\u001b[0m=\u001b[35m<Task 'task_transform_df' 0x155b60a90 (id: 4)>\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:10.341425\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mFailed to retrieve table from local table cache\u001b[0m [\u001b[0m\u001b[1m\u001b[34mParquetTableCache\u001b[0m]\u001b[0m \u001b[36mcause\u001b[0m=\u001b[35mInvalid type <class 'pydiverse.transform.eager.pandas_table.PandasTableImpl'>\u001b[0m \u001b[36mtable\u001b[0m=\u001b[35m<Table 'titanic' (t1_raw_input)>\u001b[0m\n",
      "/Users/martin/micromamba/envs/vectorization/lib/python3.11/site-packages/duckdb_engine/__init__.py:173: DuckDBEngineWarning: duckdb-engine doesn't yet support reflection on indices\n",
      "  warnings.warn(\n",
      "/Users/martin/micromamba/envs/vectorization/lib/python3.11/site-packages/pydiverse/transform/eager/pandas_table.py:244: FutureWarning: DataFrameGroupBy.grouper is deprecated and will be removed in a future version of pandas.\n",
      "  grouper = self.grouped_df().grouper if self.grouped_by else None\n",
      "\u001b[2m2024-08-06 09:44:10.486508\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mWriting table 't2_transformed_data__even.transform_df'\u001b[0m [\u001b[0m\u001b[1m\u001b[34mDuckDBTableStore\u001b[0m]\u001b[0m\n",
      "    [\u001b[36mtable_obj\u001b[0m]\n",
      "    \u001b[35m   age_bucket  samples  survival_likelyhood\n",
      "    \u001b[35m0        10.0       64              0.59375\n",
      "    \u001b[35m1        20.0      115             0.382609\n",
      "    \u001b[35m2        30.0      230             0.365217\n",
      "    \u001b[35m3        40.0      155             0.445161\n",
      "    \u001b[35m4        50.0       86             0.383721\n",
      "    \u001b[35m5        60.0       42             0.404762\n",
      "    \u001b[35m6        70.0       17             0.235294\n",
      "    \u001b[35m7        80.0        5                  0.2\n",
      "    \u001b[35m8        <NA>        0             0.293785\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:10.494748\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mTask finished successfully    \u001b[0m [\u001b[0m\u001b[1m\u001b[34mTask 'task_transform_df'\u001b[0m]\u001b[0m \u001b[36mstate\u001b[0m=\u001b[35m<FinalTaskState.COMPLETED: 1>\u001b[0m \u001b[36mtask\u001b[0m=\u001b[35m<Task 'task_transform_df' 0x155b60a90 (id: 4)>\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:10.496754\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mFailed to retrieve table from local table cache\u001b[0m [\u001b[0m\u001b[1m\u001b[34mParquetTableCache\u001b[0m]\u001b[0m \u001b[36mcause\u001b[0m=\u001b[35mCan't retrieve Table as type <class 'pydiverse.transform.lazy.sql_table.sql_table.SQLTableImpl'>. This is either because no TableHook has been registered for this type, or because not all requirements have been met for the corresponding hook.\n",
      "Hooks with unmet requirements: pydiverse.pipedag.backend.table.sql.hooks.TidyPolarsTableHook\u001b[0m \u001b[36mtable\u001b[0m=\u001b[35m<Table 'titanic' (t1_raw_input)>\u001b[0m\n",
      "/Users/martin/micromamba/envs/vectorization/lib/python3.11/site-packages/duckdb_engine/__init__.py:173: DuckDBEngineWarning: duckdb-engine doesn't yet support reflection on indices\n",
      "  warnings.warn(\n",
      "\u001b[2m2024-08-06 09:44:10.624836\u001b[0m [\u001b[33m\u001b[1mwarning  \u001b[0m] \u001b[1mCache miss                    \u001b[0m [\u001b[0m\u001b[1m\u001b[34mDuckDBTableStore\u001b[0m]\u001b[0m \u001b[36mcause\u001b[0m=\u001b[35mNo result found for lazy table cache key\u001b[0m \u001b[36mstage\u001b[0m=\u001b[35mt2_transformed_data\u001b[0m \u001b[36mtable\u001b[0m=\u001b[35mtransform_sql\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:10.626181\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mExecuting sql                 \u001b[0m [\u001b[0m\u001b[1m\u001b[34mDuckDBTableStore\u001b[0m]\u001b[0m\n",
      "    [\u001b[36mquery\u001b[0m]\n",
      "    \u001b[35mCREATE TABLE t2_transformed_data__even.transform_sql AS\n",
      "    \u001b[35mSELECT transform_sql.age_bucket AS age_bucket, transform_sql.samples AS samples, transform_sql.survival_likelyhood AS survival_likelyhood \n",
      "    \u001b[35mFROM (SELECT ROUND(titanic.age + 4.999, -1) AS age_bucket, count(ROUND(titanic.age + 4.999, -1)) AS samples, AVG(titanic.survived) AS survival_likelyhood \n",
      "    \u001b[35mFROM t1_raw_input.titanic AS titanic GROUP BY ROUND(titanic.age + 4.999, -1) ORDER BY ROUND(titanic.age + 4.999, -1) ASC NULLS LAST) AS transform_sql\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:10.632154\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mTask finished successfully    \u001b[0m [\u001b[0m\u001b[1m\u001b[34mTask 'task_transform_sql'\u001b[0m]\u001b[0m \u001b[36mstate\u001b[0m=\u001b[35m<FinalTaskState.COMPLETED: 1>\u001b[0m \u001b[36mtask\u001b[0m=\u001b[35m<Task 'task_transform_sql' 0x1065f4990 (id: 5)>\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:10.633626\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mFailed to retrieve table from local table cache\u001b[0m [\u001b[0m\u001b[1m\u001b[34mParquetTableCache\u001b[0m]\u001b[0m \u001b[36mcause\u001b[0m=\u001b[35mCan't retrieve Table as type <class 'ibis.expr.types.relations.Table'>. This is either because no TableHook has been registered for this type, or because not all requirements have been met for the corresponding hook.\n",
      "Hooks with unmet requirements: pydiverse.pipedag.backend.table.sql.hooks.TidyPolarsTableHook\u001b[0m \u001b[36mtable\u001b[0m=\u001b[35m<Table 'titanic' (t1_raw_input)>\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:10.662086\u001b[0m [\u001b[33m\u001b[1mwarning  \u001b[0m] \u001b[1mCache miss                    \u001b[0m [\u001b[0m\u001b[1m\u001b[34mDuckDBTableStore\u001b[0m]\u001b[0m \u001b[36mcause\u001b[0m=\u001b[35mNo result found for lazy table cache key\u001b[0m \u001b[36mstage\u001b[0m=\u001b[35mt2_transformed_data\u001b[0m \u001b[36mtable\u001b[0m=\u001b[35mtask_ibis_vz4vcva2ozcxtyt3i37p_0000\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:10.705147\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mExecuting sql                 \u001b[0m [\u001b[0m\u001b[1m\u001b[34mDuckDBTableStore\u001b[0m]\u001b[0m\n",
      "    [\u001b[36mquery\u001b[0m]\n",
      "    \u001b[35mCREATE TABLE t2_transformed_data__even.task_ibis_vz4vcva2ozcxtyt3i37p_0000 AS\n",
      "    \u001b[35mWITH t0 AS (\n",
      "    \u001b[35m  SELECT\n",
      "    \u001b[35m    t2.survived AS survived,\n",
      "    \u001b[35m    t2.pclass AS pclass,\n",
      "    \u001b[35m    t2.sex AS sex,\n",
      "    \u001b[35m    t2.age AS age,\n",
      "    \u001b[35m    t2.sibsp AS sibsp,\n",
      "    \u001b[35m    t2.parch AS parch,\n",
      "    \u001b[35m    t2.fare AS fare,\n",
      "    \u001b[35m    t2.embarked AS embarked,\n",
      "    \u001b[35m    t2.class AS class,\n",
      "    \u001b[35m    t2.who AS who,\n",
      "    \u001b[35m    t2.adult_male AS adult_male,\n",
      "    \u001b[35m    t2.deck AS deck,\n",
      "    \u001b[35m    t2.embark_town AS embark_town,\n",
      "    \u001b[35m    t2.alive AS alive,\n",
      "    \u001b[35m    t2.alone AS alone,\n",
      "    \u001b[35m    ROUND(t2.age + CAST(4.999 AS DECIMAL(18, 3)), CAST(-1 AS TINYINT)) AS age_bucket\n",
      "    \u001b[35m  FROM t1_raw_input.titanic AS t2\n",
      "    \u001b[35m)\n",
      "    \u001b[35mSELECT\n",
      "    \u001b[35m  t1.age_bucket,\n",
      "    \u001b[35m  t1.samples,\n",
      "    \u001b[35m  t1.survival_likelyhood\n",
      "    \u001b[35mFROM (\n",
      "    \u001b[35m  SELECT\n",
      "    \u001b[35m    t0.age_bucket AS age_bucket,\n",
      "    \u001b[35m    COUNT(t0.age_bucket) AS samples,\n",
      "    \u001b[35m    AVG(t0.survived) AS survival_likelyhood\n",
      "    \u001b[35m  FROM t0\n",
      "    \u001b[35m  GROUP BY\n",
      "    \u001b[35m    1\n",
      "    \u001b[35m) AS t1\n",
      "    \u001b[35mORDER BY\n",
      "    \u001b[35m  t1.age_bucket ASC\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:10.710771\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mTask finished successfully    \u001b[0m [\u001b[0m\u001b[1m\u001b[34mTask 'task_ibis'\u001b[0m]\u001b[0m \u001b[36mstate\u001b[0m=\u001b[35m<FinalTaskState.COMPLETED: 1>\u001b[0m \u001b[36mtask\u001b[0m=\u001b[35m<Task 'task_ibis' 0x1065f4a10 (id: 6)>\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:10.712584\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mFailed to retrieve table from local table cache\u001b[0m [\u001b[0m\u001b[1m\u001b[34mParquetTableCache\u001b[0m]\u001b[0m \u001b[36mcause\u001b[0m=\u001b[35mCan't retrieve Table as type <class 'sqlalchemy.sql.schema.Table'>. This is either because no TableHook has been registered for this type, or because not all requirements have been met for the corresponding hook.\n",
      "Hooks with unmet requirements: pydiverse.pipedag.backend.table.sql.hooks.TidyPolarsTableHook\u001b[0m \u001b[36mtable\u001b[0m=\u001b[35m<Table 'titanic' (t1_raw_input)>\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:10.843422\u001b[0m [\u001b[33m\u001b[1mwarning  \u001b[0m] \u001b[1mCache miss                    \u001b[0m [\u001b[0m\u001b[1m\u001b[34mDuckDBTableStore\u001b[0m]\u001b[0m \u001b[36mcause\u001b[0m=\u001b[35mNo result found for lazy table cache key\u001b[0m \u001b[36mstage\u001b[0m=\u001b[35mt2_transformed_data\u001b[0m \u001b[36mtable\u001b[0m=\u001b[35mtask_sqlalchemy_w4wadategregxdnowbbn_0000\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:10.844838\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mExecuting sql                 \u001b[0m [\u001b[0m\u001b[1m\u001b[34mDuckDBTableStore\u001b[0m]\u001b[0m\n",
      "    [\u001b[36mquery\u001b[0m]\n",
      "    \u001b[35mCREATE TABLE t2_transformed_data__even.task_sqlalchemy_w4wadategregxdnowbbn_0000 AS\n",
      "    \u001b[35mSELECT round(titanic.age + 4.999, -1) AS age_bucket, count(round(titanic.age + 4.999, -1)) AS samples, avg(titanic.survived) AS survival_likelyhood \n",
      "    \u001b[35mFROM t1_raw_input.titanic AS titanic GROUP BY round(titanic.age + 4.999, -1) ORDER BY age_bucket\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:10.849768\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mTask finished successfully    \u001b[0m [\u001b[0m\u001b[1m\u001b[34mTask 'task_sqlalchemy'\u001b[0m]\u001b[0m \u001b[36mstate\u001b[0m=\u001b[35m<FinalTaskState.COMPLETED: 1>\u001b[0m \u001b[36mtask\u001b[0m=\u001b[35m<Task 'task_sqlalchemy' 0x1065f5650 (id: 7)>\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:10.851581\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mFailed to retrieve table from local table cache\u001b[0m [\u001b[0m\u001b[1m\u001b[34mParquetTableCache\u001b[0m]\u001b[0m \u001b[36mcause\u001b[0m=\u001b[35mCan't retrieve Table as type <class 'sqlalchemy.sql.schema.Table'>. This is either because no TableHook has been registered for this type, or because not all requirements have been met for the corresponding hook.\n",
      "Hooks with unmet requirements: pydiverse.pipedag.backend.table.sql.hooks.TidyPolarsTableHook\u001b[0m \u001b[36mtable\u001b[0m=\u001b[35m<Table 'titanic' (t1_raw_input)>\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:10.979254\u001b[0m [\u001b[33m\u001b[1mwarning  \u001b[0m] \u001b[1mCache miss                    \u001b[0m [\u001b[0m\u001b[1m\u001b[34mDuckDBTableStore\u001b[0m]\u001b[0m \u001b[36mcause\u001b[0m=\u001b[35mNo result found for lazy table cache key\u001b[0m \u001b[36mstage\u001b[0m=\u001b[35mt2_transformed_data\u001b[0m \u001b[36mtable\u001b[0m=\u001b[35mtask_sql_j3362cymhyyu3gjkc2l6_0000\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:10.980493\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mExecuting sql                 \u001b[0m [\u001b[0m\u001b[1m\u001b[34mDuckDBTableStore\u001b[0m]\u001b[0m\n",
      "    [\u001b[36mquery\u001b[0m]\n",
      "    \u001b[35mCREATE TABLE t2_transformed_data__even.task_sql_j3362cymhyyu3gjkc2l6_0000 AS\n",
      "\n",
      "    \u001b[35m        SELECT round((titanic.age + 4.999) / CAST(10 AS NUMERIC)) * 10        AS age_bucket,\n",
      "    \u001b[35m               count(round((titanic.age + 4.999) / CAST(10 AS NUMERIC)) * 10) AS samples,\n",
      "    \u001b[35m               AVG(titanic.survived)                                          AS survival_likelyhood\n",
      "    \u001b[35m        FROM t1_raw_input.titanic AS titanic\n",
      "    \u001b[35m        GROUP BY round((titanic.age + 4.999) / CAST(10 AS NUMERIC)) * 10\n",
      "    \u001b[35m        ORDER BY round((titanic.age + 4.999) / CAST(10 AS NUMERIC)) * 10 ASC NULLS LAST\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:10.985541\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mTask finished successfully    \u001b[0m [\u001b[0m\u001b[1m\u001b[34mTask 'task_sql'\u001b[0m]\u001b[0m \u001b[36mstate\u001b[0m=\u001b[35m<FinalTaskState.COMPLETED: 1>\u001b[0m \u001b[36mtask\u001b[0m=\u001b[35m<Task 'task_sql' 0x1065f5690 (id: 8)>\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:10.988283\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mFailed to retrieve task from cache\u001b[0m [\u001b[0m\u001b[1m\u001b[34mTask 'print_tables'\u001b[0m]\u001b[0m \u001b[36mcause\u001b[0m=\u001b[35mCouldn't retrieve task from cache: <Task 'print_tables' 0x1065f5910 (id: 9)>\u001b[0m \u001b[36mtask\u001b[0m=\u001b[35m<Task 'print_tables' 0x1065f5910 (id: 9)>\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:10.990209\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mRetrieved table from local table cache\u001b[0m [\u001b[0m\u001b[1m\u001b[34mParquetTableCache\u001b[0m]\u001b[0m \u001b[36mtable\u001b[0m=\u001b[35m<Table 'task_pandas_j63nc4stdmsqfsxeg3r4_0000' (t2_transformed_data)>\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:10.991601\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mRetrieved table from local table cache\u001b[0m [\u001b[0m\u001b[1m\u001b[34mParquetTableCache\u001b[0m]\u001b[0m \u001b[36mtable\u001b[0m=\u001b[35m<Table 'task_polars_d42ck7w3oge7b2vp44x6_0000' (t2_transformed_data)>\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:10.992879\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mRetrieved table from local table cache\u001b[0m [\u001b[0m\u001b[1m\u001b[34mParquetTableCache\u001b[0m]\u001b[0m \u001b[36mtable\u001b[0m=\u001b[35m<Table 'transform_df' (t2_transformed_data)>\u001b[0m\n",
      "/Users/martin/micromamba/envs/vectorization/lib/python3.11/site-packages/duckdb_engine/__init__.py:173: DuckDBEngineWarning: duckdb-engine doesn't yet support reflection on indices\n",
      "  warnings.warn(\n",
      "/Users/martin/micromamba/envs/vectorization/lib/python3.11/site-packages/duckdb_engine/__init__.py:173: DuckDBEngineWarning: duckdb-engine doesn't yet support reflection on indices\n",
      "  warnings.warn(\n",
      "/Users/martin/micromamba/envs/vectorization/lib/python3.11/site-packages/duckdb_engine/__init__.py:173: DuckDBEngineWarning: duckdb-engine doesn't yet support reflection on indices\n",
      "  warnings.warn(\n",
      "/Users/martin/micromamba/envs/vectorization/lib/python3.11/site-packages/duckdb_engine/__init__.py:173: DuckDBEngineWarning: duckdb-engine doesn't yet support reflection on indices\n",
      "  warnings.warn(\n",
      "\u001b[2m2024-08-06 09:44:11.514776\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mTask finished successfully    \u001b[0m [\u001b[0m\u001b[1m\u001b[34mTask 'print_tables'\u001b[0m]\u001b[0m \u001b[36mstate\u001b[0m=\u001b[35m<FinalTaskState.COMPLETED: 1>\u001b[0m \u001b[36mtask\u001b[0m=\u001b[35m<Task 'print_tables' 0x1065f5910 (id: 9)>\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:11.515490\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mCommitting stage              \u001b[0m [\u001b[0m\u001b[1m\u001b[34mCommit Stage\u001b[0m]\u001b[0m \u001b[36mstage\u001b[0m=\u001b[35m<Stage: t2_transformed_data>\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:11.524991\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mExecuting sql                 \u001b[0m [\u001b[0m\u001b[1m\u001b[34mDuckDBTableStore\u001b[0m]\u001b[0m\n",
      "    [\u001b[36mquery\u001b[0m]\n",
      "    \u001b[35mDROP VIEW t2_transformed_data.task_ibis_vz4vcva2ozcxtyt3i37p_0000\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:11.525383\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mExecuting sql                 \u001b[0m [\u001b[0m\u001b[1m\u001b[34mDuckDBTableStore\u001b[0m]\u001b[0m\n",
      "    [\u001b[36mquery\u001b[0m]\n",
      "    \u001b[35mDROP VIEW t2_transformed_data.task_polars_7po2t6jb5anlgrgzwa3z_0000\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:11.525733\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mExecuting sql                 \u001b[0m [\u001b[0m\u001b[1m\u001b[34mDuckDBTableStore\u001b[0m]\u001b[0m\n",
      "    [\u001b[36mquery\u001b[0m]\n",
      "    \u001b[35mDROP VIEW t2_transformed_data.task_sqlalchemy_w4wadategregxdnowbbn_0000\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:11.526142\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mExecuting sql                 \u001b[0m [\u001b[0m\u001b[1m\u001b[34mDuckDBTableStore\u001b[0m]\u001b[0m\n",
      "    [\u001b[36mquery\u001b[0m]\n",
      "    \u001b[35mDROP VIEW t2_transformed_data.task_sql_j3362cymhyyu3gjkc2l6_0000\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:11.526477\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mExecuting sql                 \u001b[0m [\u001b[0m\u001b[1m\u001b[34mDuckDBTableStore\u001b[0m]\u001b[0m\n",
      "    [\u001b[36mquery\u001b[0m]\n",
      "    \u001b[35mDROP VIEW t2_transformed_data.titanic\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:11.526778\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mExecuting sql                 \u001b[0m [\u001b[0m\u001b[1m\u001b[34mDuckDBTableStore\u001b[0m]\u001b[0m\n",
      "    [\u001b[36mquery\u001b[0m]\n",
      "    \u001b[35mDROP VIEW t2_transformed_data.transform_df\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:11.527088\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mExecuting sql                 \u001b[0m [\u001b[0m\u001b[1m\u001b[34mDuckDBTableStore\u001b[0m]\u001b[0m\n",
      "    [\u001b[36mquery\u001b[0m]\n",
      "    \u001b[35mDROP VIEW t2_transformed_data.transform_sql\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:11.531567\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mExecuting sql                 \u001b[0m [\u001b[0m\u001b[1m\u001b[34mDuckDBTableStore\u001b[0m]\u001b[0m\n",
      "    [\u001b[36mquery\u001b[0m]\n",
      "    \u001b[35mCREATE VIEW t2_transformed_data.task_ibis_vz4vcva2ozcxtyt3i37p_0000 AS\n",
      "    \u001b[35mSELECT * \n",
      "    \u001b[35mFROM t2_transformed_data__even.task_ibis_vz4vcva2ozcxtyt3i37p_0000\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:11.532483\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mExecuting sql                 \u001b[0m [\u001b[0m\u001b[1m\u001b[34mDuckDBTableStore\u001b[0m]\u001b[0m\n",
      "    [\u001b[36mquery\u001b[0m]\n",
      "    \u001b[35mCREATE VIEW t2_transformed_data.task_pandas_j63nc4stdmsqfsxeg3r4_0000 AS\n",
      "    \u001b[35mSELECT * \n",
      "    \u001b[35mFROM t2_transformed_data__even.task_pandas_j63nc4stdmsqfsxeg3r4_0000\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:11.533463\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mExecuting sql                 \u001b[0m [\u001b[0m\u001b[1m\u001b[34mDuckDBTableStore\u001b[0m]\u001b[0m\n",
      "    [\u001b[36mquery\u001b[0m]\n",
      "    \u001b[35mCREATE VIEW t2_transformed_data.task_polars_d42ck7w3oge7b2vp44x6_0000 AS\n",
      "    \u001b[35mSELECT * \n",
      "    \u001b[35mFROM t2_transformed_data__even.task_polars_d42ck7w3oge7b2vp44x6_0000\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:11.534315\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mExecuting sql                 \u001b[0m [\u001b[0m\u001b[1m\u001b[34mDuckDBTableStore\u001b[0m]\u001b[0m\n",
      "    [\u001b[36mquery\u001b[0m]\n",
      "    \u001b[35mCREATE VIEW t2_transformed_data.task_sqlalchemy_w4wadategregxdnowbbn_0000 AS\n",
      "    \u001b[35mSELECT * \n",
      "    \u001b[35mFROM t2_transformed_data__even.task_sqlalchemy_w4wadategregxdnowbbn_0000\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:11.535095\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mExecuting sql                 \u001b[0m [\u001b[0m\u001b[1m\u001b[34mDuckDBTableStore\u001b[0m]\u001b[0m\n",
      "    [\u001b[36mquery\u001b[0m]\n",
      "    \u001b[35mCREATE VIEW t2_transformed_data.task_sql_j3362cymhyyu3gjkc2l6_0000 AS\n",
      "    \u001b[35mSELECT * \n",
      "    \u001b[35mFROM t2_transformed_data__even.task_sql_j3362cymhyyu3gjkc2l6_0000\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:11.535868\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mExecuting sql                 \u001b[0m [\u001b[0m\u001b[1m\u001b[34mDuckDBTableStore\u001b[0m]\u001b[0m\n",
      "    [\u001b[36mquery\u001b[0m]\n",
      "    \u001b[35mCREATE VIEW t2_transformed_data.transform_df AS\n",
      "    \u001b[35mSELECT * \n",
      "    \u001b[35mFROM t2_transformed_data__even.transform_df\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:11.536624\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mExecuting sql                 \u001b[0m [\u001b[0m\u001b[1m\u001b[34mDuckDBTableStore\u001b[0m]\u001b[0m\n",
      "    [\u001b[36mquery\u001b[0m]\n",
      "    \u001b[35mCREATE VIEW t2_transformed_data.transform_sql AS\n",
      "    \u001b[35mSELECT * \n",
      "    \u001b[35mFROM t2_transformed_data__even.transform_sql\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:11.540756\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mTask finished successfully    \u001b[0m [\u001b[0m\u001b[1m\u001b[34mCommit Stage\u001b[0m]\u001b[0m \u001b[36mstage\u001b[0m=\u001b[35m<Stage: t2_transformed_data>\u001b[0m \u001b[36mstate\u001b[0m=\u001b[35m<FinalTaskState.COMPLETED: 1>\u001b[0m\n",
      "\u001b[2m2024-08-06 09:44:11.542073\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mFlow visualization            \u001b[0m [\u001b[0m\u001b[1m\u001b[34mFlow\u001b[0m]\u001b[0m \u001b[36murl\u001b[0m=\u001b[35mhttps://kroki.io/graphviz/svg/eNqtkk1qwzAQhfc6hXG3LTiOfzHutocoRYwsKRGZ2K4kU0LJ3WvHqomTFLSoNh-a0XujB8PVTkO_D96Cb2IGNl8aHIwVmtoN1fBFVdsPduwjMIF1eF0NK2LsCUVtOlS8ImzXdNjpOnyKLieOxhdziSE0h4rIrrWrQhS8O2MtgM-2lIOF8DmQCvHXUBRSNtlYnAeGU0_w8KMiZ_Lg6zG1GlojO30U_OJ3leC--Q9B4iWIBXOgPbQczE0I4EKW7FGI7Y26Q9D-6mStXtJRLr090r88zCd6m2RrE8WUf4h8rR3HAjZ7cTx5OxR3Dt7ScpH2WrWWWmAo_P8-7mAUvLwGcTVz65g4po6ZY-5YTHszspw2YGbimDpmjrlj4XgmP60hMbQ=\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "            samples  survival_likelyhood\n",
      "age_bucket                              \n",
      "10.0             64              0.59375\n",
      "20.0            115             0.382609\n",
      "30.0            230             0.365217\n",
      "40.0            155             0.445161\n",
      "50.0             86             0.383721\n",
      "60.0             42             0.404762\n",
      "70.0             17             0.235294\n",
      "80.0              5                  0.2\n",
      "\n",
      "\n",
      "   age_bucket  samples  survival_likelyhood\n",
      "0        <NA>        0             0.293785\n",
      "1        10.0       64              0.59375\n",
      "2        20.0      115             0.382609\n",
      "3        30.0      230             0.365217\n",
      "4        40.0      155             0.445161\n",
      "5        50.0       86             0.383721\n",
      "6        60.0       42             0.404762\n",
      "7        70.0       17             0.235294\n",
      "8        80.0        5                  0.2\n",
      "\n",
      "\n",
      "   age_bucket  samples  survival_likelyhood\n",
      "0        10.0       64              0.59375\n",
      "1        20.0      115             0.382609\n",
      "2        30.0      230             0.365217\n",
      "3        40.0      155             0.445161\n",
      "4        50.0       86             0.383721\n",
      "5        60.0       42             0.404762\n",
      "6        70.0       17             0.235294\n",
      "7        80.0        5                  0.2\n",
      "8        <NA>        0             0.293785\n",
      "\n",
      "\n",
      "   age_bucket  samples  survival_likelyhood\n",
      "0        10.0       64              0.59375\n",
      "1        20.0      115             0.382609\n",
      "2        30.0      230             0.365217\n",
      "3        40.0      155             0.445161\n",
      "4        50.0       86             0.383721\n",
      "5        60.0       42             0.404762\n",
      "6        70.0       17             0.235294\n",
      "7        80.0        5                  0.2\n",
      "8        <NA>        0             0.293785\n",
      "\n",
      "\n",
      "   age_bucket  samples  survival_likelyhood\n",
      "0        10.0       64              0.59375\n",
      "1        20.0      115             0.382609\n",
      "2        30.0      230             0.365217\n",
      "3        40.0      155             0.445161\n",
      "4        50.0       86             0.383721\n",
      "5        60.0       42             0.404762\n",
      "6        70.0       17             0.235294\n",
      "7        80.0        5                  0.2\n",
      "8        <NA>        0             0.293785\n",
      "\n",
      "\n",
      "   age_bucket  samples  survival_likelyhood\n",
      "0        10.0       64              0.59375\n",
      "1        20.0      115             0.382609\n",
      "2        30.0      230             0.365217\n",
      "3        40.0      155             0.445161\n",
      "4        50.0       86             0.383721\n",
      "5        60.0       42             0.404762\n",
      "6        70.0       17             0.235294\n",
      "7        80.0        5                  0.2\n",
      "8        <NA>        0             0.293785\n",
      "\n",
      "\n",
      "   age_bucket  samples  survival_likelyhood\n",
      "0        10.0       64              0.59375\n",
      "1        20.0      115             0.382609\n",
      "2        30.0      230             0.365217\n",
      "3        40.0      155             0.445161\n",
      "4        50.0       86             0.383721\n",
      "5        60.0       42             0.404762\n",
      "6        70.0       17             0.235294\n",
      "7        80.0        5                  0.2\n",
      "8        <NA>        0             0.293785\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[2m2024-08-06 09:44:11.746949\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mStopped IPCServer             \u001b[0m [\u001b[0m\u001b[1m\u001b[34mRunContextServer\u001b[0m]\u001b[0m \u001b[36maddress\u001b[0m=\u001b[35mtcp://127.0.0.1:56025\u001b[0m\n"
     ]
    },
    {
     "data": {
      "image/svg+xml": [
       "<svg xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\" width=\"1004pt\" height=\"257pt\" viewBox=\"0.00 0.00 1004.00 257.00\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 253)\">\n",
       "<title>G</title>\n",
       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-253 1000,-253 1000,4 -4,4\"/>\n",
       "<g id=\"clust1\" class=\"cluster\">\n",
       "<title>cluster_t1_raw_input</title>\n",
       "<polygon fill=\"#000000\" fill-opacity=\"0.125490\" stroke=\"black\" points=\"450,-164.5 450,-241 606,-241 606,-164.5 450,-164.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"528\" y=\"-223.7\" font-family=\"Times,serif\" font-size=\"14.00\">t1_raw_input</text>\n",
       "</g>\n",
       "<g id=\"clust2\" class=\"cluster\">\n",
       "<title>cluster_t2_transformed_data</title>\n",
       "<polygon fill=\"#000000\" fill-opacity=\"0.125490\" stroke=\"black\" points=\"8,-8 8,-156.5 988,-156.5 988,-8 8,-8\"/>\n",
       "<text text-anchor=\"middle\" x=\"498\" y=\"-139.2\" font-family=\"Times,serif\" font-size=\"14.00\">t2_transformed_data</text>\n",
       "</g>\n",
       "<!-- 0 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>0</title>\n",
       "<ellipse fill=\"#e8ffc6\" stroke=\"black\" cx=\"528\" cy=\"-190.5\" rx=\"70.29\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"528\" y=\"-185.45\" font-family=\"Times,serif\" font-size=\"14.00\">read_input_data</text>\n",
       "</g>\n",
       "<!-- 2 -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>2</title>\n",
       "<ellipse fill=\"#adef9b\" stroke=\"black\" cx=\"72\" cy=\"-106\" rx=\"55.96\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"72\" y=\"-100.95\" font-family=\"Times,serif\" font-size=\"14.00\">task_pandas</text>\n",
       "</g>\n",
       "<!-- 0&#45;&gt;2 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>0-&gt;2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M457.79,-188.26C361.45,-185.77 193.74,-178.35 137,-156.5 122.27,-150.83 108.07,-140.74 96.72,-131.14\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"99.2,-128.67 89.4,-124.65 94.56,-133.9 99.2,-128.67\"/>\n",
       "</g>\n",
       "<!-- 3 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>3</title>\n",
       "<ellipse fill=\"#adef9b\" stroke=\"black\" cx=\"199\" cy=\"-106\" rx=\"52.89\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"199\" y=\"-100.95\" font-family=\"Times,serif\" font-size=\"14.00\">task_polars</text>\n",
       "</g>\n",
       "<!-- 0&#45;&gt;3 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>0-&gt;3</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M460.23,-185.33C393.23,-180.46 295.82,-171.15 261,-156.5 247.11,-150.66 233.78,-140.77 223.05,-131.36\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"225.54,-128.9 215.82,-124.7 220.8,-134.05 225.54,-128.9\"/>\n",
       "</g>\n",
       "<!-- 4 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>4</title>\n",
       "<ellipse fill=\"#adef9b\" stroke=\"black\" cx=\"349\" cy=\"-106\" rx=\"78.99\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"349\" y=\"-100.95\" font-family=\"Times,serif\" font-size=\"14.00\">task_transform_df</text>\n",
       "</g>\n",
       "<!-- 0&#45;&gt;4 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>0-&gt;4</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M485.74,-175.8C470.19,-170.34 452.58,-163.65 437,-156.5 419.49,-148.47 400.79,-138.2 385.15,-129.11\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"387.06,-126.17 376.67,-124.11 383.5,-132.2 387.06,-126.17\"/>\n",
       "</g>\n",
       "<!-- 5 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>5</title>\n",
       "<ellipse fill=\"#adef9b\" stroke=\"black\" cx=\"528\" cy=\"-106\" rx=\"82.06\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"528\" y=\"-100.95\" font-family=\"Times,serif\" font-size=\"14.00\">task_transform_sql</text>\n",
       "</g>\n",
       "<!-- 0&#45;&gt;5 -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>0-&gt;5</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M528,-172C528,-161.5 528,-147.89 528,-135.83\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"531.5,-135.98 528,-125.98 524.5,-135.98 531.5,-135.98\"/>\n",
       "</g>\n",
       "<!-- 6 -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>6</title>\n",
       "<ellipse fill=\"#adef9b\" stroke=\"black\" cx=\"672\" cy=\"-106\" rx=\"43.67\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"672\" y=\"-100.95\" font-family=\"Times,serif\" font-size=\"14.00\">task_ibis</text>\n",
       "</g>\n",
       "<!-- 0&#45;&gt;6 -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>0-&gt;6</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M574.55,-176.66C589.34,-171.53 605.39,-164.84 619,-156.5 630.23,-149.62 641.13,-140.1 650.09,-131.25\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"652.53,-133.77 656.99,-124.16 647.51,-128.89 652.53,-133.77\"/>\n",
       "</g>\n",
       "<!-- 7 -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>7</title>\n",
       "<ellipse fill=\"#adef9b\" stroke=\"black\" cx=\"807\" cy=\"-106\" rx=\"72.85\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"807\" y=\"-100.95\" font-family=\"Times,serif\" font-size=\"14.00\">task_sqlalchemy</text>\n",
       "</g>\n",
       "<!-- 0&#45;&gt;7 -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>0-&gt;7</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M594.82,-184.61C633.74,-180.22 683.21,-171.97 725,-156.5 742.78,-149.92 760.99,-139.38 775.72,-129.71\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"777.28,-132.89 783.6,-124.39 773.36,-127.09 777.28,-132.89\"/>\n",
       "</g>\n",
       "<!-- 8 -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>8</title>\n",
       "<ellipse fill=\"#adef9b\" stroke=\"black\" cx=\"939\" cy=\"-106\" rx=\"41.12\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"939\" y=\"-100.95\" font-family=\"Times,serif\" font-size=\"14.00\">task_sql</text>\n",
       "</g>\n",
       "<!-- 0&#45;&gt;8 -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>0-&gt;8</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M598.17,-188.67C688.28,-186.65 839.01,-179.78 889,-156.5 900.92,-150.95 911.65,-141.32 920.09,-132.02\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"922.66,-134.4 926.46,-124.5 917.32,-129.87 922.66,-134.4\"/>\n",
       "</g>\n",
       "<!-- 9 -->\n",
       "<g id=\"node9\" class=\"node\">\n",
       "<title>9</title>\n",
       "<ellipse fill=\"#adef9b\" stroke=\"black\" cx=\"528\" cy=\"-34\" rx=\"54.42\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"528\" y=\"-28.95\" font-family=\"Times,serif\" font-size=\"14.00\">print_tables</text>\n",
       "</g>\n",
       "<!-- 2&#45;&gt;9 -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>2-&gt;9</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M113.09,-93.45C120.99,-91.46 129.22,-89.54 137,-88 251.81,-65.29 387.52,-49.29 465.11,-41.15\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"465.03,-44.68 474.62,-40.17 464.31,-37.72 465.03,-44.68\"/>\n",
       "</g>\n",
       "<!-- 3&#45;&gt;9 -->\n",
       "<g id=\"edge9\" class=\"edge\">\n",
       "<title>3-&gt;9</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M238.58,-93.69C246.01,-91.7 253.72,-89.72 261,-88 332.5,-71.13 415.61,-55.2 469.87,-45.3\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"470.29,-48.78 479.51,-43.55 469.04,-41.89 470.29,-48.78\"/>\n",
       "</g>\n",
       "<!-- 4&#45;&gt;9 -->\n",
       "<g id=\"edge10\" class=\"edge\">\n",
       "<title>4-&gt;9</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M387.4,-89.98C415.58,-78.96 453.92,-63.97 483.45,-52.42\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"484.45,-55.79 492.48,-48.89 481.9,-49.27 484.45,-55.79\"/>\n",
       "</g>\n",
       "<!-- 5&#45;&gt;9 -->\n",
       "<g id=\"edge11\" class=\"edge\">\n",
       "<title>5-&gt;9</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M528,-87.7C528,-80.41 528,-71.73 528,-63.54\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"531.5,-63.62 528,-53.62 524.5,-63.62 531.5,-63.62\"/>\n",
       "</g>\n",
       "<!-- 6&#45;&gt;9 -->\n",
       "<g id=\"edge12\" class=\"edge\">\n",
       "<title>6-&gt;9</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M644.58,-91.67C622.87,-81.12 592.25,-66.23 567.84,-54.37\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"569.69,-51.37 559.17,-50.15 566.63,-57.67 569.69,-51.37\"/>\n",
       "</g>\n",
       "<!-- 7&#45;&gt;9 -->\n",
       "<g id=\"edge13\" class=\"edge\">\n",
       "<title>7-&gt;9</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M757.12,-92.49C707.58,-80.05 631.88,-61.06 581.2,-48.35\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"582.18,-44.98 571.62,-45.95 580.47,-51.77 582.18,-44.98\"/>\n",
       "</g>\n",
       "<!-- 8&#45;&gt;9 -->\n",
       "<g id=\"edge14\" class=\"edge\">\n",
       "<title>8-&gt;9</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M908.32,-93.59C901.99,-91.51 895.33,-89.53 889,-88 786.27,-63.21 664.13,-48.22 591.5,-40.79\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"592.08,-37.33 581.78,-39.81 591.38,-44.3 592.08,-37.33\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>"
      ],
      "text/plain": [
       "<IPython.core.display.SVG object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "flow = get_pipeline()\n",
    "result = flow.run()\n",
    "result.visualize()\n",
    "assert result.successful"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "source": [
    "Next: [vectorization08.ipynb](vectorization08.ipynb): window functions"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}