diff --git a/lib/strainer/runner.rb b/lib/strainer/runner.rb
index 348ae21..5656f6d 100644
--- a/lib/strainer/runner.rb
+++ b/lib/strainer/runner.rb
@@ -5,12 +5,16 @@ def initialize(sandbox, options = {})
       @cookbooks = @sandbox.cookbooks
       @options = options
 
+      # need a variable at the root level to track whether the
+      # build actually passed
+      success = true
+
       @cookbooks.each do |cookbook|
         $stdout.puts
         $stdout.puts Color.negative{ "# Straining '#{cookbook}'" }.to_s
 
         commands_for(cookbook).collect do |command|
-          success = run(command)
+          success &= run(command)
 
           if fail_fast? && !success
             $stdout.puts [ label_with_padding(command[:label]), Color.red{ 'Exited early because --fail-fast was specified. Some tests may have been skipped!' } ].join(' ')
@@ -20,6 +24,9 @@ def initialize(sandbox, options = {})
 
         $stdout.puts
       end
+
+      # fail unless all commands returned successfully
+      abort unless success
     end
 
     private
diff --git a/strainer.gemspec b/strainer.gemspec
index 4fae750..95b5c21 100644
--- a/strainer.gemspec
+++ b/strainer.gemspec
@@ -1,5 +1,5 @@
 Gem::Specification.new do |gem|
-  gem.version       = '0.0.3'
+  gem.version       = '0.0.4'
   gem.authors       = ['Seth Vargo']
   gem.email         = ['sethvargo@gmail.com']
   gem.description   = %q{Run isolated cookbook tests against your chef repository with Strainer.}