benchmark sqlite

nanocube/__init__.py

@@ -67,6 +67,7 @@ def __init__(self, df: pd.DataFrame, dimensions: list | None = None, measures:li
             except TypeError:
                 members, records = np.unique(df[col].replace({None: ""}), return_inverse=True)
             self.bitmaps.append(dict([(m, BitMap(np.where(records == i)[0])) for i, m in enumerate(members)]))
+        pass
 
     def get(self, *args, **kwargs):
         """

research/issue_007.py

@@ -0,0 +1,37 @@
+from pandas import read_csv as pd_read_csv
+from nanocube import NanoCube
+from polars import read_csv, col
+
+def filter_with_polars():
+    df = read_csv("files/spotify_data.csv")
+    result = df.filter(col("Daily") == 1337404).select("Streams").sum()
+    print(result)
+    # shape: (1, 1)
+    # ┌────────────┐
+    # │ streams    │
+    # │ ---        │
+    # │ i64        │
+    # ╞════════════╡
+    # │ 3518744128 │
+    # └────────────┘
+
+def filter_with_nanocube():
+    df = pd_read_csv("files/spotify_data.csv")
+    # FIXME: issue #7 -> solved: Daily was used a measure by default, querying it as a dimension will return all rows.
+    # nc = NanoCube(df)
+    nc = NanoCube(df, dimensions=['Daily'], measures=['Streams'])
+    result = nc.get("Streams", Daily=1337404)
+    print(result)
+    # 2345359210015
+
+def filter_with_pandas():
+    df = pd_read_csv("files/spotify_data.csv")
+    result = df.loc[(df['Daily'] == 1337404)]['Streams'].sum()
+    print(result)
+    # 2345359210015
+
+if __name__ == "__main__":
+    filter_with_polars()
+    filter_with_nanocube()
+    filter_with_pandas()
+    # main()

research/nano_vs_duckdb.py

@@ -1,14 +1,16 @@
 from nanocube import NanoCube
 import polars as pl
 import duckdb
+import pandas as pd
 
 from timeit import timeit
 
-# Create a DataFrame
-df = pl.read_parquet('files/car_prices.parquet')
-ns = NanoCube(df.to_pandas(), dimensions=['make', 'model', 'trim', 'body'], measures=['mmr'])
-ducktable = duckdb.sql("SELECT * FROM 'files/car_prices.parquet'")
+# Create a DataFrame and NanoCube
+df = pd.read_parquet('files/car_prices.parquet')
+ns = NanoCube(df, dimensions=['make', 'model', 'trim', 'body'], measures=['mmr'])
 
+# Create a DuckDB table
+duckdb.sql("CREATE TABLE car_prices AS SELECT * FROM 'files/car_prices.parquet'")
 
 
 def query_nanocube(loops=1000):
@@ -20,7 +22,7 @@ def query_nanocube(loops=1000):
 def query_duckdb(loops=1000):
     value = 0
     for _ in range(loops):
-        value += duckdb.sql("SELECT SUM(mmr) FROM ducktable WHERE model='Optima' AND trim='LX' AND make='Kia' AND body='Sedan';").fetchall()[0][0]
+        value += duckdb.sql("SELECT SUM(mmr) FROM car_prices WHERE model='Optima' AND trim='LX' AND make='Kia' AND body='Sedan';").fetchall()[0][0]
     return value
 
 

research/nano_vs_polars.py

@@ -1,11 +1,14 @@
 from nanocube import NanoCube
+import pandas as pd
 import polars as pl
-
 from timeit import timeit
 
-# Create a DataFrame
+# Create a DataFrame and NanoCube
+df = pd.read_parquet('files/car_prices.parquet')
+ns = NanoCube(df, dimensions=['make', 'model', 'trim', 'body'], measures=['mmr'])
+
+# Create a Polars table
 df = pl.read_parquet('files/car_prices.parquet')
-ns = NanoCube(df.to_pandas(), dimensions=['make', 'model', 'trim', 'body'], measures=['mmr'])
 
 
 def query_nanocube(loops=1000):

research/nano_vs_sqlite.py

@@ -0,0 +1,45 @@
+from nanocube import NanoCube
+import pandas as pd
+import sqlite3
+from timeit import timeit
+
+
+# Create a DataFrame and NanoCube
+df = pd.read_parquet('files/car_prices.parquet')
+ns = NanoCube(df, dimensions=['make', 'model', 'trim', 'body'], measures=['mmr'])
+
+# Connect to in-memory SQLite database
+conn = sqlite3.connect(':memory:')
+df.to_sql('car_prices', conn, index=False)
+cursor = conn.cursor()
+if True:
+    cursor.execute("CREATE INDEX index_car_prices ON car_prices (make, model, trim, body);")
+
+
+def query_nanocube(loops=1000):
+    value = 0
+    for _ in range(loops):
+        value += ns.get('mmr', model='Optima', trim='LX', make='Kia', body='Sedan')
+    return value
+
+def query_sqlite(loops=1000):
+    value = 0
+    sql = "SELECT SUM(mmr) FROM car_prices WHERE model='Optima' AND trim='LX' AND make='Kia' AND body='Sedan';"
+    for _ in range(loops):
+        cursor.execute(sql)
+        result = cursor.fetchone()[0]
+        value += result
+    return value
+
+
+if __name__ == '__main__':
+    pl_time = timeit(query_sqlite, number=1)
+    nc_time = timeit(query_nanocube, number=1)
+    print(f"SQLite point query in {pl_time:.5f} sec.")
+    print(f"NanoCube point query in {nc_time:.5f} sec.")
+    print(f"NanoCube is {pl_time/nc_time:.2f}x times faster than SQLite on query with 4 filters on 1 measure:")
+    print(f"\tns.get('mmr', model='Optima', trim='LX', make='Kia', body='Sedan')")
+    assert(query_nanocube() == query_sqlite())
+
+    # Close the connection
+    conn.close()