Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: fails transfer if free space below thresh #478

Open
wants to merge 4 commits into
base: transition-to-runkit
Choose a base branch
from
+76 −19
Open

feat: fails transfer if free space below thresh #478

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
2bd3efb
feat: fails transfer if free space below thresh
eduardacoppo Mar 12, 2025
ab1b9e7
chore: make transfer threshold a cli arg
eduardacoppo Mar 13, 2025
70e4ee0
chore: rename threshold and specify it in MBs
eduardacoppo Mar 14, 2025
086bdb6
fix: check space in target_dir, not root
eduardacoppo Mar 14, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 20 additions & 7 deletions lib/syskit/cli/log_runtime_archive.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,13 +43,16 @@ def initialize(
# through FTP server
#
# @param [Params] server_params the FTP server parameters
# #param [Integer] min_free_space the minimum required space
# in the root_dir to perform transfer
# @return [Array<TransferDatasetResult>]
def process_root_folder_transfer(server_params)
def process_root_folder_transfer(server_params, min_free_space)
candidates = self.class.find_all_dataset_folders(@root_dir)
running = candidates.last
candidates.map do |child|
process_dataset_transfer(
child, server_params, @root_dir, full: child != running
child, server_params, @root_dir, full: child != running,
thresh: min_free_space
)
end
end
Expand Down Expand Up @@ -139,9 +142,10 @@ def process_dataset(child, full:, max_archive_size: DEFAULT_MAX_ARCHIVE_SIZE)
end
end

def process_dataset_transfer(child, server, root, full:)
def process_dataset_transfer(child, server, root, full:, thresh:)
self.class.transfer_dataset(
child, server, root, full: full, logger: @logger
child, server, root,
full: full, thresh: thresh, logger: @logger
)
end

Expand All @@ -160,7 +164,7 @@ def failures
# Transfer the given dataset
def self.transfer_dataset(
dataset_path, server, root,
full:, logger: null_logger
full:, thresh:, logger: null_logger
)
logger.info(
"Transfering dataset #{dataset_path} in " \
Expand All @@ -176,7 +180,7 @@ def self.transfer_dataset(
end

transfer_results = candidates.map do |child_path|
result = transfer_file(child_path, server, root)
result = transfer_file(child_path, server, root, thresh)
child_path.unlink if result.success?

result
Expand Down Expand Up @@ -222,7 +226,16 @@ def self.log_transfer_results(dataset_path, result, logger: null_logger)
# Transfer a file to the central log server via FTP
#
# @return [LogUploadState:Result]
def self.transfer_file(file, server, root)
def self.transfer_file(file, server, root, thresh)
free_space = Sys::Filesystem.stat(root).bytes_available

if free_space < thresh
server.raise_error(
552, "Requested file transfer aborted for #{file}. " \
"Exceeded storage allocation for root dir."
)
end

ftp = RobyApp::LogTransferServer::FTPUpload.new(
server.host, server.port, server.certificate, server.user,
server.password, file,
Expand Down
8 changes: 7 additions & 1 deletion lib/syskit/cli/log_runtime_archive_main.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,9 @@ def archive(root_dir, target_dir)
type: :numeric, default: 600, desc: "polling period in seconds"
option :max_upload_rate_mbps,
type: :numeric, default: 1_000, desc: "max upload rate in Mbps"
option :min_free_space,
type: :numeric, default: 10_000_000_000,
desc: "min space in source_dir required to perform transfer"
def watch_transfer( # rubocop:disable Metrics/ParameterLists
source_dir, host, port, certificate_path, user, password, implicit_ftps
)
Expand All @@ -88,6 +91,8 @@ def watch_transfer( # rubocop:disable Metrics/ParameterLists
desc "transfer", "transfers the datasets"
option :max_upload_rate_mbps,
type: :numeric, default: 1_000, desc: "max upload rate in Mbps"
option :min_free_space, type: :numeric, default: 10_000_000_000,
desc: "min space in source_dir required to perform transfer"
def transfer( # rubocop:disable Metrics/ParameterLists
source_dir, host, port, certificate_path, user, password, implicit_ftps
)
Expand All @@ -100,7 +105,8 @@ def transfer( # rubocop:disable Metrics/ParameterLists
implicit_ftps: implicit_ftps,
max_upload_rate: rate_mbps_to_bps(options[:max_upload_rate_mbps])
)
archiver.process_root_folder_transfer(server_params)
archiver.process_root_folder_transfer(server_params,
min_free_space)
end

desc "transfer_server", "creates the log transfer FTP server \
Expand Down
38 changes: 31 additions & 7 deletions test/cli/test_log_runtime_archive.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -534,6 +534,7 @@ def should_archive_dataset(dataset, archive_basename, full:)
params.port = @server.port
@params = params
@process = LogRuntimeArchive.new(@root)
@min_free_space = 666
end

after do
Expand Down Expand Up @@ -563,7 +564,7 @@ def create_server(params)
make_random_file "test.0.log", root: dataset_b
make_random_file "test.1.log", root: dataset_b

@process.process_root_folder_transfer(@params)
@process.process_root_folder_transfer(@params, @min_free_space)

assert(File.exist?(@target_dir / "20220434-2023" / "test.0.log"))
assert(File.exist?(@target_dir / "20220434-2023" / "test.1.log"))
Expand All @@ -580,7 +581,7 @@ def create_server(params)
make_random_file "test.0.log", root: dataset
make_random_file "test.1.log", root: dataset
@process.process_dataset_transfer(
dataset, @params, @root, full: true
dataset, @params, @root, full: true, thresh: @min_free_space
)

assert(File.exist?(@target_dir / "PATH" / "test.0.log"))
Expand All @@ -593,7 +594,7 @@ def create_server(params)
make_random_file "test.1.log", root: dataset

@process.process_dataset_transfer(
dataset, @params, @root, full: true
dataset, @params, @root, full: true, thresh: @min_free_space
)

assert(
Expand All @@ -605,12 +606,13 @@ def create_server(params)
describe ".transfer_dataset" do
before do
@dataset = make_valid_folder("PATH")
@free_space_threshold = 666
make_random_file "test.0.log", root: @dataset
end

it "transfers a dataset through FTP" do
results = LogRuntimeArchive.transfer_dataset(
@dataset, @params, @root, full: true
@dataset, @params, @root, full: true, thresh: @min_free_space
)

assert results.success?
Expand All @@ -621,7 +623,7 @@ def create_server(params)

it "removes the source file if the transfer was successful" do
results = LogRuntimeArchive.transfer_dataset(
@dataset, @params, @root, full: true
@dataset, @params, @root, full: true, thresh: @min_free_space
)

assert results.success?
Expand All @@ -636,7 +638,7 @@ def create_server(params)
.should_receive(:transfer_file)
.and_return(result)
results = LogRuntimeArchive.transfer_dataset(
@dataset, @params, @root, full: true
@dataset, @params, @root, full: true, thresh: @min_free_space
)

refute results.success?
Expand All @@ -649,12 +651,34 @@ def create_server(params)
dataset = make_valid_folder("PATH")
make_random_file "test.log", root: dataset
result = LogRuntimeArchive.transfer_file(
dataset / "test.log", @params, @root
dataset / "test.log", @params, @root, @min_free_space
)

assert(File.exist?(@target_dir / "PATH" / "test.log"))
assert result.success?, "transfer failed: #{result.message}"
end

it "fails file transfer if free space is lower than threshold" do
dataset = make_valid_folder("PATH")
make_random_file "test.log", root: dataset
mock_available_space(665)

assert_raises(StandardError) do
LogRuntimeArchive.transfer_file(
dataset / "test.log", @params, @root, @min_free_space
)
end
end

def mock_available_space(free_space, directory: @root)
flexmock(Sys::Filesystem)
.should_receive(:stat).with(directory)
.and_return do
flexmock(
bytes_available: free_space
)
end
end
end
end

Expand Down