results.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv('comparisons.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>method</th>\n",
       "      <th>rows</th>\n",
       "      <th>image size mb</th>\n",
       "      <th>peak memory mb</th>\n",
       "      <th>total time min</th>\n",
       "      <th>type</th>\n",
       "      <th>Handles rotating credentials?</th>\n",
       "      <th>platform</th>\n",
       "      <th>finished</th>\n",
       "      <th>Score</th>\n",
       "      <th>notes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>pg_dump/pg_restore</td>\n",
       "      <td>41,011,955.00</td>\n",
       "      <td>145.0</td>\n",
       "      <td>34.0</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>EL</td>\n",
       "      <td>True</td>\n",
       "      <td>postgresql</td>\n",
       "      <td>True</td>\n",
       "      <td>102.000000</td>\n",
       "      <td>This is purely EL job for postgresql.  It need...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>pyspark_copy</td>\n",
       "      <td>41,011,955.00</td>\n",
       "      <td>1720.0</td>\n",
       "      <td>955.0</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>ETL</td>\n",
       "      <td>True</td>\n",
       "      <td>python</td>\n",
       "      <td>True</td>\n",
       "      <td>2865.000000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>pyspark_write</td>\n",
       "      <td>41,011,955.00</td>\n",
       "      <td>1610.0</td>\n",
       "      <td>1075.0</td>\n",
       "      <td>4.500000</td>\n",
       "      <td>ETL</td>\n",
       "      <td>True</td>\n",
       "      <td>python</td>\n",
       "      <td>True</td>\n",
       "      <td>4837.500000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>duckdb_copy</td>\n",
       "      <td>41,011,955.00</td>\n",
       "      <td>566.0</td>\n",
       "      <td>3148.0</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>ETL</td>\n",
       "      <td>True</td>\n",
       "      <td>c</td>\n",
       "      <td>True</td>\n",
       "      <td>6296.000000</td>\n",
       "      <td>Only sql</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>spark_write</td>\n",
       "      <td>41,011,955.00</td>\n",
       "      <td>1330.0</td>\n",
       "      <td>2053.0</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>ETL</td>\n",
       "      <td>True</td>\n",
       "      <td>scala</td>\n",
       "      <td>True</td>\n",
       "      <td>10265.000000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>polars_connectorx_copy</td>\n",
       "      <td>41,011,955.00</td>\n",
       "      <td>943.0</td>\n",
       "      <td>11330.0</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>ETL</td>\n",
       "      <td>True</td>\n",
       "      <td>python</td>\n",
       "      <td>True</td>\n",
       "      <td>56650.000000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>polars adbc copy</td>\n",
       "      <td>41,011,955.00</td>\n",
       "      <td>841.0</td>\n",
       "      <td>11320.0</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>ETL</td>\n",
       "      <td>True</td>\n",
       "      <td>python</td>\n",
       "      <td>True</td>\n",
       "      <td>67920.000000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>pg_dump/pg_restore</td>\n",
       "      <td>2,000,000.00</td>\n",
       "      <td>145.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>0.050000</td>\n",
       "      <td>EL</td>\n",
       "      <td>True</td>\n",
       "      <td>postgresql</td>\n",
       "      <td>True</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>This is purely EL job for postgresql.  It need...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>duckdb_copy</td>\n",
       "      <td>2,000,000.00</td>\n",
       "      <td>566.0</td>\n",
       "      <td>345.0</td>\n",
       "      <td>0.016667</td>\n",
       "      <td>ETL</td>\n",
       "      <td>True</td>\n",
       "      <td>c</td>\n",
       "      <td>True</td>\n",
       "      <td>5.750000</td>\n",
       "      <td>Only sql</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>polars adbc copy</td>\n",
       "      <td>2,000,000.00</td>\n",
       "      <td>841.0</td>\n",
       "      <td>587.0</td>\n",
       "      <td>0.050000</td>\n",
       "      <td>ETL</td>\n",
       "      <td>True</td>\n",
       "      <td>python</td>\n",
       "      <td>True</td>\n",
       "      <td>29.350000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>polars_connectorx_copy</td>\n",
       "      <td>2,000,000.00</td>\n",
       "      <td>943.0</td>\n",
       "      <td>600.0</td>\n",
       "      <td>0.050000</td>\n",
       "      <td>ETL</td>\n",
       "      <td>True</td>\n",
       "      <td>python</td>\n",
       "      <td>True</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>pyspark_copy</td>\n",
       "      <td>2,000,000.00</td>\n",
       "      <td>1720.0</td>\n",
       "      <td>809.0</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>ETL</td>\n",
       "      <td>True</td>\n",
       "      <td>python</td>\n",
       "      <td>True</td>\n",
       "      <td>67.416666</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>pyspark_write</td>\n",
       "      <td>2,000,000.00</td>\n",
       "      <td>1610.0</td>\n",
       "      <td>960.0</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>ETL</td>\n",
       "      <td>True</td>\n",
       "      <td>python</td>\n",
       "      <td>True</td>\n",
       "      <td>240.000000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>pandas read_sql_copy</td>\n",
       "      <td>2,000,000.00</td>\n",
       "      <td>693.0</td>\n",
       "      <td>400.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>ETL</td>\n",
       "      <td>True</td>\n",
       "      <td>python</td>\n",
       "      <td>True</td>\n",
       "      <td>400.000000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>spark_write</td>\n",
       "      <td>2,000,000.00</td>\n",
       "      <td>1330.0</td>\n",
       "      <td>1775.0</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>ETL</td>\n",
       "      <td>True</td>\n",
       "      <td>scala</td>\n",
       "      <td>True</td>\n",
       "      <td>887.500000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>modin_copy</td>\n",
       "      <td>2,000,000.00</td>\n",
       "      <td>1480.0</td>\n",
       "      <td>2488.0</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>ETL</td>\n",
       "      <td>True</td>\n",
       "      <td>python</td>\n",
       "      <td>True</td>\n",
       "      <td>1244.000000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>polars_connectorx_write</td>\n",
       "      <td>2,000,000.00</td>\n",
       "      <td>1140.0</td>\n",
       "      <td>1867.0</td>\n",
       "      <td>1.500000</td>\n",
       "      <td>ETL</td>\n",
       "      <td>True</td>\n",
       "      <td>python</td>\n",
       "      <td>True</td>\n",
       "      <td>2800.500000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>modin_to_sql</td>\n",
       "      <td>2,000,000.00</td>\n",
       "      <td>1480.0</td>\n",
       "      <td>10000.0</td>\n",
       "      <td>2.500000</td>\n",
       "      <td>ETL</td>\n",
       "      <td>True</td>\n",
       "      <td>python</td>\n",
       "      <td>True</td>\n",
       "      <td>25000.000000</td>\n",
       "      <td>ran out of memory. 16 GB machine.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>pandas read_sql_to_sql</td>\n",
       "      <td>2,000,000.00</td>\n",
       "      <td>693.0</td>\n",
       "      <td>7700.0</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>ETL</td>\n",
       "      <td>True</td>\n",
       "      <td>python</td>\n",
       "      <td>True</td>\n",
       "      <td>38500.000000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>modin_to_sql</td>\n",
       "      <td>41,011,955.00</td>\n",
       "      <td>1480.0</td>\n",
       "      <td>12410.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ETL</td>\n",
       "      <td>True</td>\n",
       "      <td>python</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ran out of memory. 16 GB machine.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>polars_connectorx_write</td>\n",
       "      <td>41,011,955.00</td>\n",
       "      <td>1140.0</td>\n",
       "      <td>13450.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ETL</td>\n",
       "      <td>True</td>\n",
       "      <td>python</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ran out of memory. 16 GB machine.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>modin_copy</td>\n",
       "      <td>41,011,955.00</td>\n",
       "      <td>1480.0</td>\n",
       "      <td>13600.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ETL</td>\n",
       "      <td>True</td>\n",
       "      <td>python</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ran out of memory. 16 GB machine.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>pandas read_sql_copy</td>\n",
       "      <td>41,011,955.00</td>\n",
       "      <td>693.0</td>\n",
       "      <td>13960.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ETL</td>\n",
       "      <td>True</td>\n",
       "      <td>python</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ran out of memory. 16 GB machine.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>pandas read_sql_to_sql</td>\n",
       "      <td>41,011,955.00</td>\n",
       "      <td>693.0</td>\n",
       "      <td>14000.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ETL</td>\n",
       "      <td>True</td>\n",
       "      <td>python</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ran out of memory. 16 GB machine.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>datafusion</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>only reads csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>ballista/datafusion</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>only reads parquet</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     method           rows  image size mb  peak memory mb  \\\n",
       "0        pg_dump/pg_restore  41,011,955.00          145.0            34.0   \n",
       "2              pyspark_copy  41,011,955.00         1720.0           955.0   \n",
       "3             pyspark_write  41,011,955.00         1610.0          1075.0   \n",
       "1               duckdb_copy  41,011,955.00          566.0          3148.0   \n",
       "4               spark_write  41,011,955.00         1330.0          2053.0   \n",
       "5    polars_connectorx_copy  41,011,955.00          943.0         11330.0   \n",
       "6          polars adbc copy  41,011,955.00          841.0         11320.0   \n",
       "7        pg_dump/pg_restore   2,000,000.00          145.0            20.0   \n",
       "8               duckdb_copy   2,000,000.00          566.0           345.0   \n",
       "9          polars adbc copy   2,000,000.00          841.0           587.0   \n",
       "10   polars_connectorx_copy   2,000,000.00          943.0           600.0   \n",
       "11             pyspark_copy   2,000,000.00         1720.0           809.0   \n",
       "13            pyspark_write   2,000,000.00         1610.0           960.0   \n",
       "12     pandas read_sql_copy   2,000,000.00          693.0           400.0   \n",
       "14              spark_write   2,000,000.00         1330.0          1775.0   \n",
       "15               modin_copy   2,000,000.00         1480.0          2488.0   \n",
       "16  polars_connectorx_write   2,000,000.00         1140.0          1867.0   \n",
       "18             modin_to_sql   2,000,000.00         1480.0         10000.0   \n",
       "17   pandas read_sql_to_sql   2,000,000.00          693.0          7700.0   \n",
       "19             modin_to_sql  41,011,955.00         1480.0         12410.0   \n",
       "20  polars_connectorx_write  41,011,955.00         1140.0         13450.0   \n",
       "21               modin_copy  41,011,955.00         1480.0         13600.0   \n",
       "22     pandas read_sql_copy  41,011,955.00          693.0         13960.0   \n",
       "23   pandas read_sql_to_sql  41,011,955.00          693.0         14000.0   \n",
       "24               datafusion            NaN            NaN             NaN   \n",
       "25      ballista/datafusion            NaN            NaN             NaN   \n",
       "\n",
       "    total time min type Handles rotating credentials?    platform  finished  \\\n",
       "0         3.000000   EL                          True  postgresql      True   \n",
       "2         3.000000  ETL                          True      python      True   \n",
       "3         4.500000  ETL                          True      python      True   \n",
       "1         2.000000  ETL                          True           c      True   \n",
       "4         5.000000  ETL                          True       scala      True   \n",
       "5         5.000000  ETL                          True      python      True   \n",
       "6         6.000000  ETL                          True      python      True   \n",
       "7         0.050000   EL                          True  postgresql      True   \n",
       "8         0.016667  ETL                          True           c      True   \n",
       "9         0.050000  ETL                          True      python      True   \n",
       "10        0.050000  ETL                          True      python      True   \n",
       "11        0.083333  ETL                          True      python      True   \n",
       "13        0.250000  ETL                          True      python      True   \n",
       "12        1.000000  ETL                          True      python      True   \n",
       "14        0.500000  ETL                          True       scala      True   \n",
       "15        0.500000  ETL                          True      python      True   \n",
       "16        1.500000  ETL                          True      python      True   \n",
       "18        2.500000  ETL                          True      python      True   \n",
       "17        5.000000  ETL                          True      python      True   \n",
       "19             NaN  ETL                          True      python     False   \n",
       "20             NaN  ETL                          True      python     False   \n",
       "21             NaN  ETL                          True      python     False   \n",
       "22             NaN  ETL                          True      python     False   \n",
       "23             NaN  ETL                          True      python     False   \n",
       "24             NaN  NaN                           NaN         NaN     False   \n",
       "25             NaN  NaN                           NaN         NaN     False   \n",
       "\n",
       "           Score                                              notes  \n",
       "0     102.000000  This is purely EL job for postgresql.  It need...  \n",
       "2    2865.000000                                                NaN  \n",
       "3    4837.500000                                                NaN  \n",
       "1    6296.000000                                           Only sql  \n",
       "4   10265.000000                                                NaN  \n",
       "5   56650.000000                                                NaN  \n",
       "6   67920.000000                                                NaN  \n",
       "7       1.000000  This is purely EL job for postgresql.  It need...  \n",
       "8       5.750000                                           Only sql  \n",
       "9      29.350000                                                NaN  \n",
       "10     30.000000                                                NaN  \n",
       "11     67.416666                                                NaN  \n",
       "13    240.000000                                                NaN  \n",
       "12    400.000000                                                NaN  \n",
       "14    887.500000                                                NaN  \n",
       "15   1244.000000                                                NaN  \n",
       "16   2800.500000                                                NaN  \n",
       "18  25000.000000                  ran out of memory. 16 GB machine.  \n",
       "17  38500.000000                                                NaN  \n",
       "19           NaN                  ran out of memory. 16 GB machine.  \n",
       "20           NaN                  ran out of memory. 16 GB machine.  \n",
       "21           NaN                  ran out of memory. 16 GB machine.  \n",
       "22           NaN                  ran out of memory. 16 GB machine.  \n",
       "23           NaN                  ran out of memory. 16 GB machine.  \n",
       "24           NaN                                     only reads csv  \n",
       "25           NaN                                 only reads parquet  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = data.assign(Score = lambda x: x['peak memory mb'] * x['total time min']).sort_values(['finished', 'rows', 'Score'], ascending=[False, False, True])\n",
    "data"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "etl-mQdwFmza",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}