benchmark sqlite

nanocubeai · Oct 7, 2024 · 27e24bb · 27e24bb
1 parent 87ffe26
commit 27e24bb
Show file tree

Hide file tree

Showing 8 changed files with 2,519 additions and 10 deletions.
diff --git a/research/files/car_prices.parquet → benchmarks/files/car_prices.parquet b/research/files/car_prices.parquet → benchmarks/files/car_prices.parquet
diff --git a/research/files/df.parquet → benchmarks/files/df.parquet b/research/files/df.parquet → benchmarks/files/df.parquet
diff --git a/research/files/nanocube.parquet → benchmarks/files/nanocube.parquet b/research/files/nanocube.parquet → benchmarks/files/nanocube.parquet
diff --git a/research/nano_vs_duckdb.py → benchmarks/nano_vs_duckdb.py b/research/nano_vs_duckdb.py → benchmarks/nano_vs_duckdb.py
@@ -1,16 +1,18 @@
 from nanocube import NanoCube
-import polars as pl
 import duckdb
 import pandas as pd
-
 from timeit import timeit
+from pathlib import Path
+import os
+
 
 # Create a DataFrame and NanoCube
-df = pd.read_parquet('files/car_prices.parquet')
+file_car_prices = Path(os.path.dirname(os.path.realpath(__file__))) / "files" / "car_prices.parquet"
+df = pd.read_parquet(file_car_prices)
 ns = NanoCube(df, dimensions=['make', 'model', 'trim', 'body'], measures=['mmr'])
 
 # Create a DuckDB table
-duckdb.sql("CREATE TABLE car_prices AS SELECT * FROM 'files/car_prices.parquet'")
+duckdb.sql(f"CREATE TABLE car_prices AS SELECT * FROM '{file_car_prices}'")
 
 
 def query_nanocube(loops=1000):

diff --git a/research/nano_vs_polars.py → benchmarks/nano_vs_polars.py b/research/nano_vs_polars.py → benchmarks/nano_vs_polars.py
@@ -2,13 +2,16 @@
 import pandas as pd
 import polars as pl
 from timeit import timeit
+from pathlib import Path
+import os
 
 # Create a DataFrame and NanoCube
-df = pd.read_parquet('files/car_prices.parquet')
+file_car_prices = Path(os.path.dirname(os.path.realpath(__file__))) / "files" / "car_prices.parquet"
+df = pd.read_parquet(file_car_prices)
 ns = NanoCube(df, dimensions=['make', 'model', 'trim', 'body'], measures=['mmr'])
 
 # Create a Polars table
-df = pl.read_parquet('files/car_prices.parquet')
+df = pl.read_parquet(file_car_prices)
 
 
 def query_nanocube(loops=1000):

diff --git a/research/nano_vs_sqlite.py → benchmarks/nano_vs_sqlite.py b/research/nano_vs_sqlite.py → benchmarks/nano_vs_sqlite.py
@@ -2,10 +2,13 @@
 import pandas as pd
 import sqlite3
 from timeit import timeit
+from pathlib import Path
+import os
 
 
 # Create a DataFrame and NanoCube
-df = pd.read_parquet('files/car_prices.parquet')
+file_car_prices = Path(os.path.dirname(os.path.realpath(__file__))) / "files" / "car_prices.parquet"
+df = pd.read_parquet(file_car_prices)
 ns = NanoCube(df, dimensions=['make', 'model', 'trim', 'body'], measures=['mmr'])
 
 # Connect to in-memory SQLite database

diff --git a/research/files/spotify_data.csv b/research/files/spotify_data.csv
diff --git a/research/serialize.py b/research/serialize.py
@@ -100,21 +100,21 @@ def deserialize_nano(file_name) -> NanoCube:
 
     # Deserialize DataFrame from Parquet
     start = datetime.now()
-    df2 = pd.read_parquet('files/df.parquet')
+    df2 = pd.read_parquet('../benchmarks/files/df.parquet')
     duration = (datetime.now() - start).total_seconds()
     print(f"Deserialized dataframe from Parquet in {duration:.5f} sec.")
 
     nc = NanoCube(df)
 
     # Serialize NanoCube to Parquet
     start = datetime.now()
-    serialize_nano(nc, 'files/nanocube.parquet')
+    serialize_nano(nc, '../benchmarks/files/nanocube.parquet')
     duration = (datetime.now() - start).total_seconds()
     print(f"Serialized NanoCube to Parquet in {duration:.5f} sec.")
 
     # Deserialize NanoCube from Parquet
     start = datetime.now()
-    nc2 = deserialize_nano('files/nanocube.parquet')
+    nc2 = deserialize_nano('../benchmarks/files/nanocube.parquet')
     duration = (datetime.now() - start).total_seconds()
     print(f"Deserialized NanoCube to Parquet in {duration:.5f} sec.")