feat(matrix): use views to create matrix query

Author: bethesque

DEVELOPER_DOCUMENTATION.md

@@ -55,3 +55,7 @@ Domain classes are found in `lib/pact_broker/domain`. Many of these classes are
 * `latest_tagged_pact_publications` - This view has the same columns as `all_pact_publications`, plus a `tag_name` column. It is used to return the pact for the latest tagged version of a consumer.
 
 * `latest_verifications` - The most recent verification for each pact version.
+
+* `matrix` - The matrix of every pact publication and verification. Includes every pact revision (eg. publishing to the same consumer version twice, or using PATCH) and every verification (including 'overwritten' ones. eg. when the same provider build runs twice.)
+
+* `latest_matrix` - This view is a subset of, and has the same columns as, the `matrix`. It removes 'overwritten' pacts and verifications from the matrix (ie. only show latest pact revision for each consumer version and latest verification for each provider version)

db/migrations/000048_create_matrix.rb

@@ -1,6 +1,9 @@
 Sequel.migration do
   up do
-    p = :latest_pact_publications_by_consumer_versions
+    # Includes every pact revision (eg. publishing to the same consumer version twice,
+    # or using PATCH) and every verification
+    # (including 'overwritten' ones. eg. when the same provider build runs twice.)
+    p = :all_pact_publications
     create_view(:matrix,
       from(p)
         .select(
@@ -12,15 +15,16 @@
           Sequel[p][:id].as(:pact_publication_id),
           Sequel[p][:pact_version_id],
           Sequel[p][:pact_version_sha],
-          Sequel[p][:revision_number],
+          Sequel[p][:revision_number].as(:pact_revision_number),
           Sequel[p][:created_at].as(:pact_created_at),
           Sequel[p][:provider_id],
           Sequel[p][:provider_name],
           Sequel[:versions][:id].as(:provider_version_id),
           Sequel[:versions][:number].as(:provider_version_number),
           Sequel[:versions][:order].as(:provider_version_order),
+          Sequel[:verifications][:id].as(:verification_id),
           Sequel[:verifications][:success],
-          Sequel[:verifications][:number],
+          Sequel[:verifications][:number].as(:verification_number),
           Sequel[:verifications][:id].as(:verification_id),
           Sequel[:verifications][:execution_date].as(:verification_executed_at),
           Sequel[:verifications][:build_url].as(:verification_build_url)

db/migrations/000049_create_latest_verifications_for_cv_and_pv.rb

@@ -0,0 +1,17 @@
+Sequel.migration do
+  change do
+    # joining with latest_pact_publication_revision_numbers gets rid of the overwritten
+    # pact revisions, and the max(verification_id) gets rid of the overwritten
+    # verifications
+    create_view(:latest_verification_id_for_consumer_version_and_provider_version,
+      "select consumer_version_id, provider_version_id, max(verification_id) as latest_verification_id
+      from matrix
+      inner join latest_pact_publication_revision_numbers lr
+        on matrix.consumer_id = lr.consumer_id
+        and matrix.provider_id = lr.provider_id
+        and matrix.consumer_version_order = lr.consumer_version_order
+        and matrix.pact_revision_number = lr.latest_revision_number
+        group by consumer_version_id, provider_version_id"
+    )
+  end
+end

db/migrations/000050_create_latest_matrix.rb

@@ -0,0 +1,20 @@
+Sequel.migration do
+  change do
+    # Removes 'overwritten' pacts and verifications from the matrix
+    # (ie. only show latest pact revision for each consumer version and
+    # latest verification for each provider version)
+    # Must include lines where verification_id is null so that we don't
+    # lose the unverified pacts.
+    create_view(:latest_matrix,
+      "SELECT matrix.* FROM matrix
+      INNER JOIN latest_verification_id_for_consumer_version_and_provider_version AS lv
+      ON ((matrix.consumer_version_id = lv.consumer_version_id)
+      AND (matrix.provider_version_id = lv.provider_version_id)
+      AND ((matrix.verification_id = lv.latest_verification_id)))
+
+      UNION
+
+      select * from matrix where verification_id is null"
+    )
+  end
+end

lib/pact_broker/api/decorators/matrix_decorator.rb

@@ -118,7 +118,7 @@ def verification_hash(line, base_url)
               verifiedAt: line[:verification_executed_at].to_datetime.xmlschema,
               _links: {
                 self: {
-                  href: verification_url(OpenStruct.new(line), base_url)
+                  href: verification_url(OpenStruct.new(line.merge(number: line[:verification_number])), base_url)
                 }
               }
             }

lib/pact_broker/matrix/latest_row.rb

@@ -0,0 +1,12 @@
+require 'pact_broker/matrix/row'
+
+module PactBroker
+  module Matrix
+
+    # Latest pact revision for each consumer version => latest verification
+
+    class LatestRow < Row
+      set_dataset(:latest_matrix)
+    end
+  end
+end

lib/pact_broker/matrix/repository.rb

@@ -1,5 +1,6 @@
 require 'pact_broker/repositories/helpers'
 require 'pact_broker/matrix/row'
+require 'pact_broker/matrix/latest_row'
 
 module PactBroker
   module Matrix
@@ -18,7 +19,7 @@ class Repository
       def find selectors, options = {}
         # The group with the nil provider_version_numbers will be the results of the left outer join
         # that don't have verifications, so we need to include them all.
-        lines = find_all(selectors)
+        lines = find_all(selectors, options)
         lines = apply_scope(options, selectors, lines)
 
         if options.key?(:success)
@@ -33,13 +34,11 @@ def all_versions_specified? selectors
       end
 
       def apply_scope options, selectors, lines
-        return lines unless options[:latestby]
+        return lines unless options[:latestby] == 'cvp' || options[:latestby] == 'cp'
 
         group_by_columns = case options[:latestby]
         when 'cvp' then GROUP_BY_PROVIDER
         when 'cp' then GROUP_BY_PACT
-        else
-          GROUP_BY_PROVIDER_VERSION_NUMBER
         end
 
         lines.group_by{|line| group_by_columns.collect{|key| line[key] }}
@@ -50,25 +49,22 @@ def apply_scope options, selectors, lines
 
       def find_for_consumer_and_provider pacticipant_1_name, pacticipant_2_name
         selectors = [{ pacticipant_name: pacticipant_1_name }, { pacticipant_name: pacticipant_2_name }]
-        find_all(selectors)
+        find_all(selectors, {latestby: 'cvpv'})
           .sort{|l1, l2| l2[:consumer_version_order] <=> l1[:consumer_version_order]}
       end
 
       def find_compatible_pacticipant_versions selectors
-        find(selectors)
-          .group_by{|line| GROUP_BY_PROVIDER_VERSION_NUMBER.collect{|key| line[key] }}
-          .values
-          .collect{ | lines | lines.first[:provider_version_number].nil? ? lines : lines.last }
-          .flatten
+
+        find(selectors, latestby: 'cvpv')
           .select{|line| line[:success] }
       end
 
       ##
       # If the version is nil, it means all versions for that pacticipant are to be included
       #
-      def find_all selectors
+      def find_all selectors, options
         selectors = look_up_versions_for_tags(selectors)
-        query = Row.select_all
+        query = base_table(options).select_all
 
         if selectors.size == 1
           query = where_consumer_or_provider_is(selectors.first, query)
@@ -80,6 +76,11 @@ def find_all selectors
           .collect(&:values)
       end
 
+      def base_table(options)
+        return Row unless options[:latestby]
+        return LatestRow
+      end
+
       def look_up_versions_for_tags(selectors)
         selectors.collect do | selector |
           # resource validation currently stops tag being specified without latest=true

lib/pact_broker/matrix/row.rb

@@ -8,6 +8,10 @@ class Row < Sequel::Model
       dataset_module do
         include PactBroker::Repositories::Helpers
       end
+
+      def summary
+        "#{consumer_name}#{consumer_version_number} #{provider_name}#{provider_version_number || '?'} (r#{pact_revision_number}n#{verification_number || '?'})"
+      end
     end
   end
 end

script/seed-matrix.rb

@@ -42,8 +42,9 @@
 TestDataBuilder.new
   .create_pact_with_hierarchy("A", "1", "B")
   .create_verification(provider_version: '1', success: false)
-  .create_verification(provider_version: '2', number: 2)
-  .create_verification(provider_version: '4', number: 3)
+  .create_verification(provider_version: '1', number: 2, success: true)
+  .create_verification(provider_version: '2', number: 3)
+  .create_verification(provider_version: '4', number: 4)
   .create_provider_version("5")
   .use_consumer("B")
   .use_consumer_version("1")
@@ -55,17 +56,23 @@
   .create_verification(provider_version: '2', success: true)
   .create_consumer_version("3")
   .create_pact
-  .create_pact_with_hierarchy("the-example-application", "391c43cae8c0e83c570c191f7324fccd67e53abc", "another-example-application")
-  .create_verification(provider_version: '391c43cae8c0e83c570c191f7324fccd67e53abc')
-  .create_verification(provider_version: '57fa24e44efc4d8aa42bb855a8217f145b5b1b5b', number: 2, success: false)
-  .create_verification(provider_version: '4', number: 3)
-  .use_consumer("another-example-application")
-  .use_consumer_version("391c43cae8c0e83c570c191f7324fccd67e53abc")
-  .create_provider("a-third-example-application")
+  .use_consumer("A")
+  .create_consumer_version("2")
+  .use_provider("B")
   .create_pact
-  .create_verification(provider_version: '391c43cae8c0e83c570c191f7324fccd67e53abc', success: false)
-  .use_consumer_version("57fa24e44efc4d8aa42bb855a8217f145b5b1b5b")
-  .create_pact
-  .create_verification(provider_version: '57fa24e44efc4d8aa42bb855a8217f145b5b1b5b', success: true)
+  .create_verification(provider_version: '5')
+
+  # .create_pact_with_hierarchy("the-example-application", "391c43cae8c0e83c570c191f7324fccd67e53abc", "another-example-application")
+  # .create_verification(provider_version: '391c43cae8c0e83c570c191f7324fccd67e53abc')
+  # .create_verification(provider_version: '57fa24e44efc4d8aa42bb855a8217f145b5b1b5b', number: 2, success: false)
+  # .create_verification(provider_version: '4', number: 3)
+  # .use_consumer("another-example-application")
+  # .use_consumer_version("391c43cae8c0e83c570c191f7324fccd67e53abc")
+  # .create_provider("a-third-example-application")
+  # .create_pact
+  # .create_verification(provider_version: '391c43cae8c0e83c570c191f7324fccd67e53abc', success: false)
+  # .use_consumer_version("57fa24e44efc4d8aa42bb855a8217f145b5b1b5b")
+  # .create_pact
+  # .create_verification(provider_version: '57fa24e44efc4d8aa42bb855a8217f145b5b1b5b', success: true)
 
 

spec/lib/pact_broker/api/decorators/matrix_decorator_spec.rb

@@ -18,8 +18,8 @@ module Decorators
               provider_version_number: "4.5.6",
               provider_name: "Provider",
               success: line_1_success,
-              number: 1,
-              build_url: nil,
+              verification_number: 1,
+              verification_build_url: nil,
               verification_executed_at: verification_date
             }
           end
@@ -33,8 +33,8 @@ module Decorators
               provider_version_number: nil,
               provider_name: "Provider",
               success: line_2_success,
-              number: nil,
-              build_url: nil,
+              verification_number: nil,
+              verification_build_url: nil,
               verification_executed_at: verification_date
             }
           end

spec/lib/pact_broker/matrix/repository_spec.rb

@@ -12,7 +12,7 @@ def build_selectors(hash)
       end
 
       def shorten_row row
-        "#{row[:consumer_name]}#{row[:consumer_version_number]} #{row[:provider_name]}#{row[:provider_version_number] || '?'} n#{row[:number] || '?'}"
+        "#{row[:consumer_name]}#{row[:consumer_version_number]} #{row[:provider_name]}#{row[:provider_version_number] || '?'} n#{row[:verification_number] || '?'}"
       end
 
       def shorten_rows rows
@@ -23,6 +23,7 @@ def shorten_rows rows
         before do
           # A1 - B1
           # A1 - B1 r2
+          # A1 - B2 r3
           # A1 - C1
           # A2 - B?
           # A2 - C2
@@ -46,7 +47,8 @@ def shorten_rows rows
         let(:a1_b1_n1) { "A1 B1 n1" }
         let(:a1_b1_n2) { "A1 B1 n2" }
         let(:a1_b2_n3) { "A1 B2 n3" }
-        let(:a1_c1_n1) { "A1 C1 n1"}
+        let(:a1_c1_n1) { "A1 C1 n1" }
+        let(:a2_b__n_) { "A2 B? n?" }
 
         context "when just the consumer name is specified" do
           let(:selectors) { build_selectors('A' => nil) }
@@ -56,6 +58,7 @@ def shorten_rows rows
               expect(subject).to include a1_b1_n1
               expect(subject).to include a1_b1_n2
               expect(subject).to include a1_c1_n1
+              expect(subject).to include a2_b__n_
               expect(subject.size).to eq 6
             end
           end
@@ -67,6 +70,7 @@ def shorten_rows rows
               expect(subject).to_not include a1_b1_n1
               expect(subject).to include a1_b1_n2
               expect(subject).to include a1_c1_n1
+              expect(subject).to include a2_b__n_
               expect(subject.size).to eq 5
             end
           end
@@ -79,6 +83,7 @@ def shorten_rows rows
               expect(subject).to_not include a1_b1_n2
               expect(subject).to include a1_b2_n3
               expect(subject).to include a1_c1_n1
+              expect(subject).to include a2_b__n_
               expect(subject.size).to eq 4
             end
           end
@@ -554,8 +559,8 @@ def shorten_rows rows
 
       describe "#find_compatible_pacticipant_versions" do
         let(:td) { TestDataBuilder.new }
-        # subject { Repository.new.find_compatible_pacticipant_versions(selectors) }
-        subject { Repository.new.find(selectors, success: [true], scope: 'latest')}
+
+        subject { Repository.new.find(selectors, success: [true], latestby: 'cvpv')}
 
         context "when compatible versions can be found" do
           before do
@@ -582,22 +587,23 @@ def shorten_rows rows
             expect(subject.first[:consumer_version_number]).to eq "1"
             expect(subject.first[:provider_name]).to eq "B"
             expect(subject.first[:provider_version_number]).to eq "2"
-            expect(subject.first[:number]).to eq 2
+            expect(subject.first[:verification_number]).to eq 2
             expect(subject.first[:pact_created_at]).to be_datey
             expect(subject.first[:verification_executed_at]).to be_datey
 
             expect(subject.last[:consumer_name]).to eq "B"
             expect(subject.last[:consumer_version_number]).to eq "2"
             expect(subject.last[:provider_name]).to eq "C"
             expect(subject.last[:provider_version_number]).to eq "2"
-            expect(subject.last[:number]).to eq 1
+            expect(subject.last[:verification_number]).to eq 1
             expect(subject.last[:pact_created_at]).to be_datey
 
             expect(subject.size).to eq 2
           end
 
           context "when one or more pacticipants does not have a version specified" do
             let(:selectors){ build_selectors("A" => "1", "B" => "2", "C" => nil) }
+            let(:options) { { latestby: 'cvpv'} }
 
             it "returns all the rows for that pacticipant" do
               expect(subject).to include_hash_matching(provider_name: "C", provider_version_number: "2")
@@ -628,7 +634,7 @@ def shorten_rows rows
 
           it "returns the last line" do
             expect(subject.size).to eq 1
-            expect(subject.first[:number]).to eq 2
+            expect(subject).to include_hash_matching verification_number: 2
           end
         end