Skip to content

Commit

Permalink
log total number of rows fetched so far
Browse files Browse the repository at this point in the history
  • Loading branch information
pacman82 committed Nov 12, 2023
1 parent cfca854 commit 09f30bb
Show file tree
Hide file tree
Showing 4 changed files with 13 additions and 4 deletions.
2 changes: 1 addition & 1 deletion Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "odbc2parquet"
version = "3.0.1"
version = "3.1.0"
authors = ["Markus Klein"]
edition = "2021"
repository = "https://github.com/pacman82/odbc2parquet"
Expand Down
4 changes: 4 additions & 0 deletions Changelog.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
# Changelog

## 3.1.0

* Additional log message emitting the number of total rows fetched so far.

## 3.0.1

* Fix: `--no-empty-file` now works correctly with options causing files to be splitted like `--file-size-threshold` or `--row-groups-per-file`.
Expand Down
9 changes: 7 additions & 2 deletions src/query.rs
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,9 @@ fn cursor_to_parquet(

let mut pb = ParquetBuffer::new(batch_size_row);
let mut num_batch = 0;
// Count the number of total rows fetched so far for logging. This should be identical to
// `num_batch * batch_size_row + num_rows`.
let mut total_rows_fetched = 0;

let mut writer = parquet_output(path, parquet_schema.clone(), parquet_format_options)?;

Expand All @@ -169,9 +172,11 @@ fn cursor_to_parquet(
{
let mut row_group_writer = writer.next_row_group(num_batch)?;
let mut col_index = 0;
num_batch += 1;
let num_rows = buffer.num_rows();
info!("Fetched batch {} with {} rows.", num_batch, num_rows);
total_rows_fetched += num_rows;
num_batch += 1;
info!("Fetched batch {num_batch} with {num_rows} rows.");
info!("Fetched {total_rows_fetched} rows in total.");
pb.set_num_rows_fetched(num_rows);
while let Some(mut column_writer) = row_group_writer.next_column()? {
let col_name = parquet_schema.get_fields()[col_index]
Expand Down

0 comments on commit 09f30bb

Please sign in to comment.