Skip to content

Commit

Permalink
Fix test concurrency
Browse files Browse the repository at this point in the history
  • Loading branch information
@tustvold
tustvold committed Apr 9, 2023
1 parent 5514e58 commit f2ee3d8
Show file tree
Hide file tree
Showing 4 changed files with 83 additions and 82 deletions.
46 changes: 29 additions & 17 deletions object_store/src/aws/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1314,28 +1314,40 @@ mod tests {

#[tokio::test]
async fn s3_test() {
let config = maybe_skip_integration!();
let is_local = matches!(&config.endpoint, Some(e) if e.starts_with("http://"));
let integration = config.build().unwrap();
let builder = maybe_skip_integration!();
let is_local = matches!(&builder.endpoint, Some(e) if e.starts_with("http://"));

// Localstack doesn't support listing with spaces https://github.com/localstack/localstack/issues/6328
put_get_delete_list_opts(&integration, is_local).await;
list_uses_directories_correctly(&integration).await;
list_with_delimiter(&integration).await;
rename_and_copy(&integration).await;
stream_get(&integration).await;
let test = |integration| async move {
// Localstack doesn't support listing with spaces https://github.com/localstack/localstack/issues/6328
put_get_delete_list_opts(&integration, is_local).await;
list_uses_directories_correctly(&integration).await;
list_with_delimiter(&integration).await;
rename_and_copy(&integration).await;
stream_get(&integration).await;
};

let (handle, shutdown) = dedicated_tokio();

let integration = builder.clone().build().unwrap();
handle.block_on(test(integration));

// run integration test with unsigned payload enabled
let config = maybe_skip_integration!().with_unsigned_payload(true);
let is_local = matches!(&config.endpoint, Some(e) if e.starts_with("http://"));
let integration = config.build().unwrap();
put_get_delete_list_opts(&integration, is_local).await;
let integration = builder.clone().with_unsigned_payload(true).build().unwrap();
handle.block_on(test(integration));

// run integration test with checksum set to sha256
let config = maybe_skip_integration!().with_checksum_algorithm(Checksum::SHA256);
let is_local = matches!(&config.endpoint, Some(e) if e.starts_with("http://"));
let integration = config.build().unwrap();
put_get_delete_list_opts(&integration, is_local).await;
let integration = builder
.clone()
.with_checksum_algorithm(Checksum::SHA256)
.build()
.unwrap();
handle.block_on(test(integration));

// run integration test without tokio runtime
let integration = builder.with_tokio_runtime(handle).build().unwrap();
futures::executor::block_on(test(integration));

shutdown();
}

#[tokio::test]
Expand Down
33 changes: 15 additions & 18 deletions object_store/src/azure/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1137,30 +1137,27 @@ mod tests {
}};
}

#[test]
fn azure_blob_non_tokio() {
let (handle, shutdown) = dedicated_tokio();
let config = maybe_skip_integration!();
let integration = config.with_tokio_runtime(handle).build().unwrap();
futures::executor::block_on(async move {
#[tokio::test]
async fn azure_blob_test() {
let builder = maybe_skip_integration!();
let test = |integration| async move {
put_get_delete_list_opts(&integration, false).await;
list_uses_directories_correctly(&integration).await;
list_with_delimiter(&integration).await;
rename_and_copy(&integration).await;
copy_if_not_exists(&integration).await;
stream_get(&integration).await;
});
shutdown();
}
};

#[tokio::test]
async fn azure_blob_test() {
let integration = maybe_skip_integration!().build().unwrap();
put_get_delete_list_opts(&integration, false).await;
list_uses_directories_correctly(&integration).await;
list_with_delimiter(&integration).await;
rename_and_copy(&integration).await;
copy_if_not_exists(&integration).await;
stream_get(&integration).await;
let (handle, shutdown) = dedicated_tokio();

let integration = builder.clone().build().unwrap();
handle.block_on(test(integration));

let integration = builder.with_tokio_runtime(handle).build().unwrap();
futures::executor::block_on(test(integration));

shutdown();
}

// test for running integration test against actual blob service with service principal
Expand Down
41 changes: 26 additions & 15 deletions object_store/src/gcp/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1275,22 +1275,33 @@ mod test {
shutdown();
}

#[tokio::test]
async fn gcs_test() {
let integration = maybe_skip_integration!().build().unwrap();
#[test]
fn gcs_test() {
let builder = maybe_skip_integration!();
let test = |integration: GoogleCloudStorage| async move {
put_get_delete_list(&integration).await;
list_uses_directories_correctly(&integration).await;
list_with_delimiter(&integration).await;
rename_and_copy(&integration).await;
if integration.client.base_url == default_gcs_base_url() {
// Fake GCS server doesn't currently honor ifGenerationMatch
// https://github.com/fsouza/fake-gcs-server/issues/994
copy_if_not_exists(&integration).await;
// Fake GCS server does not yet implement XML Multipart uploads
// https://github.com/fsouza/fake-gcs-server/issues/852
stream_get(&integration).await;
}
};

put_get_delete_list(&integration).await;
list_uses_directories_correctly(&integration).await;
list_with_delimiter(&integration).await;
rename_and_copy(&integration).await;
if integration.client.base_url == default_gcs_base_url() {
// Fake GCS server doesn't currently honor ifGenerationMatch
// https://github.com/fsouza/fake-gcs-server/issues/994
copy_if_not_exists(&integration).await;
// Fake GCS server does not yet implement XML Multipart uploads
// https://github.com/fsouza/fake-gcs-server/issues/852
stream_get(&integration).await;
}
let (handle, shutdown) = dedicated_tokio();

let integration = builder.clone().build().unwrap();
handle.block_on(test(integration));

let integration = builder.with_tokio_runtime(handle).build().unwrap();
futures::executor::block_on(test(integration));

shutdown();
}

#[tokio::test]
Expand Down
45 changes: 13 additions & 32 deletions object_store/src/http/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -283,8 +283,8 @@ mod tests {

use super::*;

#[tokio::test]
async fn http_test() {
#[test]
fn http_test() {
dotenv::dotenv().ok();
let force = std::env::var("TEST_INTEGRATION");
if force.is_err() {
Expand All @@ -293,45 +293,26 @@ mod tests {
}
let url = std::env::var("HTTP_URL").expect("HTTP_URL must be set");
let options = ClientOptions::new().with_allow_http(true);
let integration = HttpBuilder::new()
let builder = HttpBuilder::new()
.with_url(url)
.with_client_options(options)
.build()
.unwrap();

put_get_delete_list_opts(&integration, false).await;
list_uses_directories_correctly(&integration).await;
list_with_delimiter(&integration).await;
rename_and_copy(&integration).await;
copy_if_not_exists(&integration).await;
}
.with_client_options(options);

#[test]
fn http_non_tokio() {
let (handle, shutdown) = dedicated_tokio();

dotenv::dotenv().ok();
let force = std::env::var("TEST_INTEGRATION");
if force.is_err() {
eprintln!("skipping HTTP integration test - set TEST_INTEGRATION to run");
return;
}
let url = std::env::var("HTTP_URL").expect("HTTP_URL must be set");
let options = ClientOptions::new().with_allow_http(true);
let integration = HttpBuilder::new()
.with_url(url)
.with_client_options(options)
.with_tokio_runtime(handle)
.build()
.unwrap();

futures::executor::block_on(async move {
let test = |integration| async move {
put_get_delete_list_opts(&integration, false).await;
list_uses_directories_correctly(&integration).await;
list_with_delimiter(&integration).await;
rename_and_copy(&integration).await;
copy_if_not_exists(&integration).await;
});
};

let integration = builder.clone().build().unwrap();
handle.block_on(test(integration));

let integration = builder.with_tokio_runtime(handle).build().unwrap();
futures::executor::block_on(test(integration));

shutdown();
}
}

0 comments on commit f2ee3d8

Please sign in to comment.