feat: remove refresh of materialized_matrix and materialized_head_matrix

Author: bethesque

db/migrations/20180517_create_latest_pact_publication_ids.rb

@@ -4,7 +4,7 @@
     # Keeping track of this in a table rather than having to calculate the
     # latest revision speeds things up.
     # We don't have to worry about updating it if a revision is deleted, because
-    # you can't delete a single revision through the API - all the revisions
+    # you can't delete a single pact revision through the API - all the revisions
     # for a pact are deleted together when you delete the pact resource for that
     # consumer version, and when that happens, this row will cascade delete.
     create_table(:latest_pact_publication_ids_by_consumer_versions, charset: 'utf8') do

db/migrations/20180521_create_latest_verification_ids.rb

@@ -3,6 +3,8 @@
     # Latest verification_id for each pact version/provider version.
     # Keeping track of this in a table rather than having to calculate the
     # latest revision speeds queries up.
+    # There is no way to delete an individual verification result yet, but when there
+    # is, we'll need to re-calculate the latest.
     create_table(:latest_verification_id_for_pact_version_and_provider_version, charset: 'utf8') do
       foreign_key :pact_version_id, :pact_versions, nil: false, on_delete: :cascade
       foreign_key :provider_version_id, :versions, nil: false, on_delete: :cascade

lib/pact_broker/api/resources/base_resource.rb

@@ -39,9 +39,6 @@ def update_matrix_after_request?
         end
 
         def finish_request
-          if update_matrix_after_request?
-            matrix_service.refresh(identifier_from_path)
-          end
           PactBroker.configuration.after_resource.call(self)
         end
 
@@ -153,10 +150,6 @@ def contract_validation_errors? contract, params
           invalid
         end
 
-        def with_matrix_refresh &block
-          matrix_service.refresh(identifier_from_path, &block)
-        end
-
         def find_pacticipant name, role
           pacticipant_service.find_pacticipant_by_name(name).tap do | pacticipant |
             set_json_error_message("No #{role} with name '#{name}' found") if pacticipant.nil?

lib/pact_broker/api/resources/pact.rb

@@ -87,9 +87,7 @@ def to_html
         end
 
         def delete_resource
-          with_matrix_refresh do
-            pact_service.delete(pact_params)
-          end
+          pact_service.delete(pact_params)
           true
         end
 

lib/pact_broker/api/resources/pacticipant.rb

@@ -45,9 +45,7 @@ def resource_exists?
         end
 
         def delete_resource
-          with_matrix_refresh do
-            pacticipant_service.delete pacticipant_name
-          end
+          pacticipant_service.delete pacticipant_name
           true
         end
 

lib/pact_broker/api/resources/tag.rb

@@ -22,7 +22,6 @@ def from_json
             @tag = tag_service.create identifier_from_path
             # Make it return a 201 by setting the Location header
             response.headers["Location"] = tag_url(base_url, tag)
-            matrix_service.refresh_tags(identifier_from_path)
           end
           response.body = to_json
         end
@@ -40,9 +39,7 @@ def tag
         end
 
         def delete_resource
-          matrix_service.refresh_tags(identifier_from_path) do
-            tag_service.delete identifier_from_path
-          end
+          tag_service.delete identifier_from_path
           true
         end
       end

lib/pact_broker/api/resources/version.rb

@@ -25,9 +25,7 @@ def to_json
         end
 
         def delete_resource
-          with_matrix_refresh do
-            version_service.delete version
-          end
+          version_service.delete version
           true
         end
 

lib/pact_broker/db/clean.rb

@@ -14,8 +14,8 @@ def initialize database_connection, options = {}
       end
 
       def call
-        db[:verifications].where(id: db[:materialized_head_matrix].select(:verification_id)).invert.delete
-        pp_ids = db[:materialized_head_matrix].select(:pact_publication_id)
+        db[:verifications].where(id: db[:head_matrix].select(:verification_id)).invert.delete
+        pp_ids = db[:head_matrix].select(:pact_publication_id)
 
         triggered_webhook_ids = db[:triggered_webhooks].where(pact_publication_id: pp_ids).invert.select(:id)
         db[:webhook_executions].where(triggered_webhook_id: triggered_webhook_ids).delete
@@ -33,11 +33,6 @@ def call
 
         db[:tags].where(version_id: referenced_version_ids).invert.delete
         db[:versions].where(id: referenced_version_ids).invert.delete
-
-        db[:materialized_matrix].delete
-        db[:materialized_matrix].insert(db[:matrix].select_all)
-        db[:materialized_head_matrix].delete
-        db[:materialized_head_matrix].insert(db[:head_matrix].select_all)
       end
 
       private

lib/pact_broker/matrix/head_row.rb

@@ -5,7 +5,7 @@ module Matrix
     # A row for each of the overall latest pacts, and a row for each of the latest tagged pacts
     # Rows with a nil consumer_tag_name are the overall latest
     class HeadRow < Row
-      set_dataset(:materialized_head_matrix)
+      set_dataset(:head_matrix)
 
       # one_to_one :latest_verification_for_consumer_version_tag,
       #   :class => "PactBroker::Verifications::LatestVerificationForConsumerVersionTag",
@@ -29,25 +29,6 @@ class HeadRow < Row
           end
         end
       end)
-
-      dataset_module do
-        include PactBroker::Repositories::Helpers
-        include PactBroker::Logging
-
-        def refresh ids
-          return super unless ids[:tag_name]
-
-          logger.debug("Refreshing #{model.table_name} for #{ids}")
-          db = model.db
-          table_name = model.table_name
-          criteria = { consumer_id: ids[:pacticipant_id], consumer_version_tag_name: ids[:tag_name] }
-          db.transaction do
-            db[table_name].where(criteria).delete
-            new_rows = db[source_view_name].where(criteria)
-            db[table_name].insert(new_rows)
-          end
-        end
-      end
     end
   end
 end

lib/pact_broker/matrix/repository.rb

@@ -23,24 +23,6 @@ class Repository
       GROUP_BY_PROVIDER = [:consumer_name, :consumer_version_number, :provider_name]
       GROUP_BY_PACT = [:consumer_name, :provider_name]
 
-      # Use a block when the refresh is caused by a resource deletion
-      # This allows us to store the correct object ids for use afterwards
-      def refresh params
-        criteria = find_ids_for_pacticipant_names(params)
-        yield if block_given?
-        PactBroker::Matrix::Row.refresh(criteria)
-        PactBroker::Matrix::HeadRow.refresh(criteria)
-      end
-
-      # Only need to update the HeadRow table when tags change
-      # because it only changes which rows are the latest tagged ones -
-      # it doesn't change the actual values in the underlying matrix.
-      def refresh_tags params
-        criteria = find_ids_for_pacticipant_names(params)
-        yield if block_given?
-        PactBroker::Matrix::HeadRow.refresh(criteria)
-      end
-
       def find_ids_for_pacticipant_names params
         criteria  = {}
 

lib/pact_broker/matrix/row.rb

@@ -8,7 +8,7 @@
 module PactBroker
   module Matrix
 
-    class Row < Sequel::Model(:materialized_matrix)
+    class Row < Sequel::Model(:matrix)
 
       # Used when using table_print to output query results
       TP_COLS = [ :consumer_version_number, :pact_revision_number, :provider_version_number, :verification_number]
@@ -26,32 +26,6 @@ class Row < Sequel::Model(:materialized_matrix)
         include PactBroker::Repositories::Helpers
         include PactBroker::Logging
 
-        def refresh ids
-          logger.debug("Refreshing #{model.table_name} for #{ids}")
-
-          db = model.db
-          table_name = model.table_name
-
-          if ids[:pacticipant_id]
-            db.transaction do
-              db[table_name].where(consumer_id: ids[:pacticipant_id]).or(provider_id: ids[:pacticipant_id]).delete
-              new_rows = db[source_view_name].where(consumer_id: ids[:pacticipant_id]).or(provider_id: ids[:pacticipant_id]).distinct
-              db[table_name].insert(new_rows)
-            end
-          elsif ids.any?
-            accepted_columns = [:consumer_id, :consumer_name, :provider_id, :provider_name]
-            criteria = ids.reject{ |k, v| !accepted_columns.include?(k) }
-            db.transaction do
-              db[table_name].where(criteria).delete
-              db[table_name].insert(db[source_view_name].where(criteria))
-            end
-          end
-        end
-
-        def source_view_name
-          model.table_name.to_s.gsub('materialized_', '').to_sym
-        end
-
         def matching_selectors selectors
           if selectors.size == 1
             where_consumer_or_provider_is(selectors.first)

lib/pact_broker/matrix/service.rb

@@ -10,14 +10,6 @@ module Service
       extend PactBroker::Repositories
       extend PactBroker::Services
 
-      def refresh params, &block
-        matrix_repository.refresh(params, &block)
-      end
-
-      def refresh_tags params, &block
-        matrix_repository.refresh_tags(params, &block)
-      end
-
       def find selectors, options = {}
         query_results = matrix_repository.find selectors, options
         pacticipant_names = selectors.collect{ | s| s[:pacticipant_name] }

spec/lib/pact_broker/db/clean_spec.rb

@@ -44,7 +44,7 @@ module DB
 
         it "deletes rows that aren't the latest or latest tagged" do
           subject
-          expect(db[:materialized_matrix].where(consumer_version_number: "2").count).to eq 0
+          expect(db[:matrix].where(consumer_version_number: "2").count).to eq 0
         end
 
         it "deletes orphan pact_versions" do

spec/lib/pact_broker/matrix/head_row_spec.rb

@@ -57,50 +57,6 @@ module Matrix
           end
         end
       end
-      describe "refresh", migration: true do
-        before do
-          PactBroker::Database.migrate
-        end
-
-        let(:td) { TestDataBuilder.new(auto_refresh_matrix: false) }
-
-        before do
-          td.create_pact_with_hierarchy("Foo", "1", "Bar")
-        end
-
-        context "with a consumer pacticipant_id and a consumer tag_name" do
-          before do
-            td.create_consumer_version_tag("prod")
-            Row.refresh(ids)
-          end
-          let(:ids) { { pacticipant_id: td.consumer.id, tag_name: "prod"} }
-
-          subject { HeadRow.refresh(ids) }
-
-          it "refreshes the data for the consumer and consumer tag in the head matrix" do
-            subject
-            expect(HeadRow.all.collect(&:values)).to contain_hash(provider_name: "Bar", consumer_name: "Foo", consumer_version_tag_name: "prod")
-          end
-        end
-
-        context "with a provider pacticipant_id and a provider tag_name" do
-          before do
-            td.create_verification(provider_version: "2")
-              .use_provider_version("2")
-              .create_provider_version_tag("prod")
-            Row.refresh(ids)
-          end
-
-          let(:ids) { { pacticipant_id: td.consumer.id, tag_name: "prod" } }
-
-          subject { HeadRow.refresh(ids) }
-
-          it "does not update the head matrix as the head matrix only contains consumer tags" do
-            subject
-            expect(HeadRow.count).to eq 0
-          end
-        end
-      end
     end
   end
 end

spec/lib/pact_broker/matrix/repository_spec.rb

@@ -19,26 +19,6 @@ def shorten_rows rows
         rows.collect{ |r| shorten_row(r) }
       end
 
-      describe "refresh" do
-        before do
-          td.create_pact_with_hierarchy("Foo", "1", "Bar")
-          Row.refresh(pacticipant_id: td.provider.id)
-        end
-
-        context "when deleting an object in the block" do
-          it "removes the relevant lines from the matrix" do
-            expect(Row.count).to_not eq 0
-            Repository.new.refresh(pacticipant_name: "Bar") { PactBroker::Pacticipants::Service.delete("Bar") }
-            expect(Row.count).to eq 0
-          end
-
-          it "yields the block" do
-            Repository.new.refresh(pacticipant_name: "Bar") { PactBroker::Pacticipants::Service.delete("Bar") }
-            expect(PactBroker::Domain::Pacticipant.where(name: "Bar").count).to eq 0
-          end
-        end
-      end
-
       describe "find" do
         before do
           # A1 - B1

spec/lib/pact_broker/matrix/row_spec.rb

@@ -26,62 +26,6 @@ module Matrix
         end
       end
 
-      describe "refresh", migration: true do
-        let(:td) { TestDataBuilder.new(auto_refresh_matrix: false) }
-
-        before do
-          PactBroker::Database.migrate
-          td.create_pact_with_hierarchy("Foo", "1", "Bar")
-        end
-
-        context "with only a consumer_id" do
-          subject { Row.refresh(consumer_id: td.consumer.id) }
-
-          it "refreshes the data for the consumer" do
-            subject
-            expect(Row.all.collect(&:values)).to contain_hash(provider_name: "Bar", consumer_name: "Foo")
-          end
-        end
-
-        context "with only a provider_id" do
-          subject { Row.refresh(provider_id: td.provider.id) }
-
-          it "refreshes the data for the provider" do
-            subject
-            expect(Row.all.collect(&:values)).to contain_hash(provider_name: "Bar", consumer_name: "Foo")
-          end
-        end
-
-        context "with both consumer_id and provider_id" do
-          subject { Row.refresh(provider_id: td.provider.id, consumer_id: td.consumer.id) }
-
-          it "refreshes the data for the consumer and provider" do
-            subject
-            expect(Row.all.collect(&:values)).to contain_hash(provider_name: "Bar", consumer_name: "Foo")
-          end
-        end
-
-        context "when there was existing data" do
-          it "deletes the existing data before inserting the new data" do
-            Row.refresh(provider_id: td.provider.id, consumer_id: td.consumer.id)
-            expect(Row.count).to eq 1
-            td.create_consumer_version("2")
-              .create_pact
-            Row.refresh(provider_id: td.provider.id, consumer_id: td.consumer.id)
-            expect(Row.count).to eq 2
-          end
-        end
-
-        context "with a pacticipant_id" do
-          subject { Row.refresh(pacticipant_id: td.provider.id) }
-
-          it "refreshes the data for both consumer and provider roles" do
-            subject
-            expect(Row.all.collect(&:values)).to contain_hash(provider_name: "Bar", consumer_name: "Foo")
-          end
-        end
-      end
-
       describe "<=>" do
         let(:row_1) do
           Row.new(