feat: add rake task to clean overwritten pact publications and verifications

Author: bethesque

lib/pact_broker/db/delete_overwritten_data.rb

@@ -0,0 +1,76 @@
+require 'date'
+require 'sequel'
+
+module PactBroker
+  module DB
+    class DeleteOverwrittenData
+      def self.call database_connection, options = {}
+        new(database_connection, options).call
+      end
+
+      def initialize database_connection, options = {}
+        @db = database_connection
+        @options = options
+        @before = options[:before] || DateTime.now
+      end
+
+      def call
+        deleted_counts = {}
+        kept_counts = {}
+
+
+        deleted_counts.merge!(delete_overwritten_pact_publications)
+        deleted_counts.merge!(delete_overwritten_verifications)
+        deleted_counts.merge!(delete_orphan_pact_versions)
+
+        kept_counts[:pact_publications] = db[:pact_publications].count
+        kept_counts[:verification_results] = db[:verifications].count
+        kept_counts[:pact_versions] = db[:pact_versions].count
+
+
+        { deleted: deleted_counts, kept: kept_counts }
+      end
+
+      private
+
+      attr_reader :db, :options, :before
+
+      def delete_webhook_data(triggered_webhook_ids)
+        db[:webhook_executions].where(triggered_webhook_id: triggered_webhook_ids).delete
+        db[:triggered_webhooks].where(id: triggered_webhook_ids).delete
+      end
+
+      def delete_orphan_pact_versions
+        referenced_pact_version_ids = db[:pact_publications].select(:pact_version_id).union(db[:verifications].select(:pact_version_id))
+        pact_version_ids_to_delete = db[:pact_versions].where(id: referenced_pact_version_ids).invert
+        deleted_counts = { pact_versions: pact_version_ids_to_delete.count }
+        pact_version_ids_to_delete.delete
+        deleted_counts
+      end
+
+      def delete_overwritten_pact_publications
+        pact_publication_ids_to_delete = db[:pact_publications]
+          .select(:id)
+          .where(id: db[:latest_pact_publication_ids_for_consumer_versions].select(:pact_publication_id))
+          .invert
+          .where(Sequel.lit('created_at < ?', before))
+
+        deleted_counts = { pact_publications: pact_publication_ids_to_delete.count }
+        delete_webhook_data(db[:triggered_webhooks].where(pact_publication_id: pact_publication_ids_to_delete).select(:id))
+        pact_publication_ids_to_delete.delete
+        deleted_counts
+      end
+
+      def delete_overwritten_verifications
+        verification_ids = db[:verifications].select(:id)
+          .where(id: db[:latest_verification_id_for_pact_version_and_provider_version].select(:verification_id))
+          .invert
+          .where(Sequel.lit('created_at < ?', before))
+        deleted_counts = { verification_results: verification_ids.count }
+        delete_webhook_data(db[:triggered_webhooks].where(verification_id: verification_ids).select(:id))
+        verification_ids.delete
+        deleted_counts
+      end
+    end
+  end
+end

lib/pact_broker/tasks.rb

@@ -1,4 +1,5 @@
 require 'pact_broker/tasks/migration_task'
 require 'pact_broker/tasks/data_migration_task'
+require 'pact_broker/tasks/delete_overwritten_data_task'
 require 'pact_broker/tasks/version_task'
 require 'pact_broker/tasks/clean_task'

lib/pact_broker/tasks/delete_overwritten_data_task.rb

@@ -0,0 +1,37 @@
+module PactBroker
+  module DB
+    class DeleteOverwrittenDataTask < ::Rake::TaskLib
+      attr_accessor :database_connection
+      attr_accessor :age_in_days
+
+      def initialize &block
+        rake_task &block
+      end
+
+      def rake_task &block
+        namespace :pact_broker do
+          namespace :db do
+            desc "Delete overwritten pacts and verifications from database"
+            task :delete_overwritten_data do | t, args |
+              require 'pact_broker/db/delete_overwritten_data'
+              require 'yaml'
+
+              instance_eval(&block)
+              options = {}
+
+              if age_in_days
+                options[:before] = (Date.today - age_in_days.to_i).to_datetime
+                $stdout.puts "Deleting overwritten pact publications and verifications older than #{age_in_days} days"
+              else
+                $stdout.puts "Deleting overwritten pact publications and verifications"
+              end
+
+              report = PactBroker::DB::DeleteOverwrittenData.call(database_connection, options)
+              $stdout.puts report.to_yaml
+            end
+          end
+        end
+      end
+    end
+  end
+end

spec/lib/pact_broker/db/delete_overwritten_data_spec.rb

@@ -0,0 +1,101 @@
+require 'pact_broker/db/delete_overwritten_data'
+
+module PactBroker
+  module DB
+    describe DeleteOverwrittenData do
+      describe ".call" do
+        let(:db) { PactBroker::DB.connection }
+        subject { DeleteOverwrittenData.call(db, before: before_date) }
+        let(:before_date) { nil }
+
+        context "when a pact is overwritten" do
+          let!(:pact_to_delete) { td.create_everything_for_an_integration.and_return(:pact) }
+          let!(:pact_to_keep) { td.revise_pact.and_return(:pact) }
+
+
+          it "deletes the overwritten pact" do
+            expect { subject }.to change{ db[:pact_publications].where(id: pact_to_delete.id).count }.by(-1)
+          end
+
+          it "does not delete the most recent pact" do
+            expect { subject }.to_not change{ db[:pact_publications].where(id: pact_to_keep.id).count }
+          end
+
+          it "returns a report" do
+            expect(subject[:deleted][:pact_publications]).to eq 1
+            expect(subject[:kept][:pact_publications]).to eq 1
+          end
+        end
+
+        context "when a pact has multiple verifications" do
+          let!(:verification_to_delete) do
+            td.create_pact_with_hierarchy
+              .create_verification(provider_version: "1", success: false)
+              .and_return(:verification)
+          end
+
+          let!(:verification_to_keep) { td.create_verification(provider_version: "1", number: 2, success: true).and_return(:verification) }
+
+          it "deletes the overwritten verification" do
+            expect { subject }.to change{ db[:verifications].where(id: verification_to_delete.id).count }.by(-1)
+          end
+
+          it "does not delete the most recent verification" do
+            expect { subject }.to_not change{ db[:verifications].where(id: verification_to_keep.id).count }
+          end
+
+          it "returns a report" do
+            expect(subject[:deleted][:verification_results]).to eq 1
+            expect(subject[:kept][:verification_results]).to eq 1
+          end
+        end
+
+        context "when a pact version is orphaned" do
+          before do
+            td.create_pact_with_verification.comment("this one will still have the verification, so can't be deleted")
+              .revise_pact.comment("this one can be deleted")
+              .revise_pact.comment("this one will still have a pact publication, so can't be deleted")
+          end
+
+          it "is deleted" do
+            expect { subject }.to change{ db[:pact_versions].count }.by(-1)
+          end
+
+          it "returns a report" do
+            expect(subject[:deleted][:pact_versions]).to eq 1
+            expect(subject[:kept][:pact_versions]).to eq 2
+          end
+        end
+
+        context "when the pact publication is created after the before date" do
+          before do
+            td.set_now(before_date + 1)
+              .create_pact_with_hierarchy
+              .revise_pact
+          end
+
+          let(:before_date) { DateTime.new(2010, 2, 5) }
+
+          it "doesn't delete the data" do
+            expect { subject }.to_not change { db[:pact_publications].count }
+          end
+        end
+
+        context "when the verification is created after the before date" do
+          before do
+            td.set_now(before_date + 1)
+              .create_pact_with_hierarchy
+              .create_verification(provider_version: "1", success: false)
+              .create_verification(provider_version: "1", success: true, number: 2)
+          end
+
+          let(:before_date) { DateTime.new(2010, 2, 5) }
+
+          it "doesn't delete the data" do
+            expect { subject }.to_not change { db[:verifications].count }
+          end
+        end
+      end
+    end
+  end
+end

tasks/test_db.rake

@@ -23,3 +23,10 @@ PactBroker::DB::CleanTask.new do | task |
   require 'db'
   task.database_connection = DB::PACT_BROKER_DB
 end
+
+PactBroker::DB::DeleteOverwrittenDataTask.new do | task |
+  ENV['RACK_ENV'] ||= 'test'
+  require 'db'
+  task.database_connection = DB::PACT_BROKER_DB
+  task.age_in_days = 7
+end