Skip to content

Commit

Permalink
fix --no-empty-file file naming
Browse files Browse the repository at this point in the history
  • Loading branch information
pacman82 committed Feb 22, 2024
1 parent cb28bc3 commit 99c8c61
Show file tree
Hide file tree
Showing 5 changed files with 27 additions and 33 deletions.
2 changes: 1 addition & 1 deletion Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "odbc2parquet"
version = "5.1.1"
version = "5.1.2"
authors = ["Markus Klein"]
edition = "2021"
repository = "https://github.com/pacman82/odbc2parquet"
Expand Down
5 changes: 5 additions & 0 deletions Changelog.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,10 @@
# Changelog

## 5.1.2

* Fix: 5.1.0 introduced a regression, which caused output file enumeration to happen even if file splitting is not activated, if `--no-empty-file` had been set.


## 5.1.1

* Fix: 5.1.0 introduced a regression, which caused output file enumeration to be start with a suffix of `2` instead of `1` if in addition to file splitting the `--no-empty-file` flag had also been set.
Expand Down
50 changes: 19 additions & 31 deletions src/query/parquet_writer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -104,35 +104,29 @@ impl FileWriter {
options: ParquetWriterOptions,
properties: Arc<WriterProperties>,
) -> Result<Self, Error> {
let suffix = {
if options.file_size.output_is_splitted() {
Some((1, options.suffix_length))
} else {
None
}
};

let current_path = Self::current_path(&path, suffix)?;

let (current_file, num_file) = if !options.no_empty_file {
(Some(CurrentFile::new(
current_path,
schema.clone(),
properties.clone(),
)?), 1)
} else {
(None, 0)
};

Ok(Self {
let mut file_writer = Self {
base_path: path,
schema,
properties,
file_size: options.file_size,
num_file,
num_file: 0,
suffix_length: options.suffix_length,
current_file,
})
current_file: None,
};

if !options.no_empty_file {
file_writer.next_file()?;
}

Ok(file_writer)
}

fn next_file(&mut self) -> Result<(), Error> {
let suffix = self.file_size.output_is_splitted().then_some((self.num_file + 1, self.suffix_length));
let path = Self::current_path(&self.base_path, suffix)?;
self.current_file = Some(CurrentFile::new(path, self.schema.clone(), self.properties.clone())?);
self.num_file += 1;
Ok(())
}

fn current_path(base_path: &Path, suffix: Option<(u32, usize)>) -> Result<PathBuf, Error> {
Expand All @@ -153,13 +147,7 @@ impl ParquetOutput for FileWriter {
) -> Result<(), Error> {
// There is no file. Let us create one so we can write the row group.
if self.current_file.is_none() {
// Create next file path
self.num_file += 1;
let path =
Self::current_path(&self.base_path, Some((self.num_file, self.suffix_length)))?;
let current_file =
CurrentFile::new(path, self.schema.clone(), self.properties.clone())?;
self.current_file = Some(current_file);
self.next_file()?
}

// Write next row group
Expand Down
1 change: 1 addition & 0 deletions tests/integration.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1318,6 +1318,7 @@ fn emit_file_despite_no_empty_file_set() {
"query",
"--connection-string",
MSSQL,
"--no-empty-file",
out_str,
query,
])
Expand Down

0 comments on commit 99c8c61

Please sign in to comment.