diff --git a/DEVELOPER_DOCUMENTATION.md b/DEVELOPER_DOCUMENTATION.md index 048227ba0..425f6aaa3 100644 --- a/DEVELOPER_DOCUMENTATION.md +++ b/DEVELOPER_DOCUMENTATION.md @@ -55,3 +55,7 @@ Domain classes are found in `lib/pact_broker/domain`. Many of these classes are * `latest_tagged_pact_publications` - This view has the same columns as `all_pact_publications`, plus a `tag_name` column. It is used to return the pact for the latest tagged version of a consumer. * `latest_verifications` - The most recent verification for each pact version. + +* `matrix` - The matrix of every pact publication and verification. Includes every pact revision (eg. publishing to the same consumer version twice, or using PATCH) and every verification (including 'overwritten' ones. eg. when the same provider build runs twice.) + +* `latest_matrix` - This view is a subset of, and has the same columns as, the `matrix`. It removes 'overwritten' pacts and verifications from the matrix (ie. only show latest pact revision for each consumer version and latest verification for each provider version) diff --git a/db/migrations/000048_create_matrix.rb b/db/migrations/000048_create_matrix.rb index 3126a6052..1017e26b8 100644 --- a/db/migrations/000048_create_matrix.rb +++ b/db/migrations/000048_create_matrix.rb @@ -1,6 +1,9 @@ Sequel.migration do up do - p = :latest_pact_publications_by_consumer_versions + # Includes every pact revision (eg. publishing to the same consumer version twice, + # or using PATCH) and every verification + # (including 'overwritten' ones. eg. when the same provider build runs twice.) + p = :all_pact_publications create_view(:matrix, from(p) .select( @@ -12,15 +15,16 @@ Sequel[p][:id].as(:pact_publication_id), Sequel[p][:pact_version_id], Sequel[p][:pact_version_sha], - Sequel[p][:revision_number], + Sequel[p][:revision_number].as(:pact_revision_number), Sequel[p][:created_at].as(:pact_created_at), Sequel[p][:provider_id], Sequel[p][:provider_name], Sequel[:versions][:id].as(:provider_version_id), Sequel[:versions][:number].as(:provider_version_number), Sequel[:versions][:order].as(:provider_version_order), + Sequel[:verifications][:id].as(:verification_id), Sequel[:verifications][:success], - Sequel[:verifications][:number], + Sequel[:verifications][:number].as(:verification_number), Sequel[:verifications][:id].as(:verification_id), Sequel[:verifications][:execution_date].as(:verification_executed_at), Sequel[:verifications][:build_url].as(:verification_build_url) diff --git a/db/migrations/000049_create_latest_verifications_for_cv_and_pv.rb b/db/migrations/000049_create_latest_verifications_for_cv_and_pv.rb new file mode 100644 index 000000000..04c204007 --- /dev/null +++ b/db/migrations/000049_create_latest_verifications_for_cv_and_pv.rb @@ -0,0 +1,17 @@ +Sequel.migration do + change do + # joining with latest_pact_publication_revision_numbers gets rid of the overwritten + # pact revisions, and the max(verification_id) gets rid of the overwritten + # verifications + create_view(:latest_verification_id_for_consumer_version_and_provider_version, + "select consumer_version_id, provider_version_id, max(verification_id) as latest_verification_id + from matrix + inner join latest_pact_publication_revision_numbers lr + on matrix.consumer_id = lr.consumer_id + and matrix.provider_id = lr.provider_id + and matrix.consumer_version_order = lr.consumer_version_order + and matrix.pact_revision_number = lr.latest_revision_number + group by consumer_version_id, provider_version_id" + ) + end +end diff --git a/db/migrations/000050_create_latest_matrix.rb b/db/migrations/000050_create_latest_matrix.rb new file mode 100644 index 000000000..700e87021 --- /dev/null +++ b/db/migrations/000050_create_latest_matrix.rb @@ -0,0 +1,20 @@ +Sequel.migration do + change do + # Removes 'overwritten' pacts and verifications from the matrix + # (ie. only show latest pact revision for each consumer version and + # latest verification for each provider version) + # Must include lines where verification_id is null so that we don't + # lose the unverified pacts. + create_view(:latest_matrix, + "SELECT matrix.* FROM matrix + INNER JOIN latest_verification_id_for_consumer_version_and_provider_version AS lv + ON ((matrix.consumer_version_id = lv.consumer_version_id) + AND (matrix.provider_version_id = lv.provider_version_id) + AND ((matrix.verification_id = lv.latest_verification_id))) + + UNION + + select * from matrix where verification_id is null" + ) + end +end diff --git a/lib/pact_broker/api/decorators/matrix_decorator.rb b/lib/pact_broker/api/decorators/matrix_decorator.rb index 672b250e9..25771f0f7 100644 --- a/lib/pact_broker/api/decorators/matrix_decorator.rb +++ b/lib/pact_broker/api/decorators/matrix_decorator.rb @@ -118,7 +118,7 @@ def verification_hash(line, base_url) verifiedAt: line[:verification_executed_at].to_datetime.xmlschema, _links: { self: { - href: verification_url(OpenStruct.new(line), base_url) + href: verification_url(OpenStruct.new(line.merge(number: line[:verification_number])), base_url) } } } diff --git a/lib/pact_broker/matrix/latest_row.rb b/lib/pact_broker/matrix/latest_row.rb new file mode 100644 index 000000000..fe3aca6e8 --- /dev/null +++ b/lib/pact_broker/matrix/latest_row.rb @@ -0,0 +1,12 @@ +require 'pact_broker/matrix/row' + +module PactBroker + module Matrix + + # Latest pact revision for each consumer version => latest verification + + class LatestRow < Row + set_dataset(:latest_matrix) + end + end +end diff --git a/lib/pact_broker/matrix/repository.rb b/lib/pact_broker/matrix/repository.rb index 6174f855a..f61b4b70a 100644 --- a/lib/pact_broker/matrix/repository.rb +++ b/lib/pact_broker/matrix/repository.rb @@ -1,5 +1,6 @@ require 'pact_broker/repositories/helpers' require 'pact_broker/matrix/row' +require 'pact_broker/matrix/latest_row' module PactBroker module Matrix @@ -18,7 +19,7 @@ class Repository def find selectors, options = {} # The group with the nil provider_version_numbers will be the results of the left outer join # that don't have verifications, so we need to include them all. - lines = find_all(selectors) + lines = find_all(selectors, options) lines = apply_scope(options, selectors, lines) if options.key?(:success) @@ -33,13 +34,11 @@ def all_versions_specified? selectors end def apply_scope options, selectors, lines - return lines unless options[:latestby] + return lines unless options[:latestby] == 'cvp' || options[:latestby] == 'cp' group_by_columns = case options[:latestby] when 'cvp' then GROUP_BY_PROVIDER when 'cp' then GROUP_BY_PACT - else - GROUP_BY_PROVIDER_VERSION_NUMBER end lines.group_by{|line| group_by_columns.collect{|key| line[key] }} @@ -50,25 +49,22 @@ def apply_scope options, selectors, lines def find_for_consumer_and_provider pacticipant_1_name, pacticipant_2_name selectors = [{ pacticipant_name: pacticipant_1_name }, { pacticipant_name: pacticipant_2_name }] - find_all(selectors) + find_all(selectors, {latestby: 'cvpv'}) .sort{|l1, l2| l2[:consumer_version_order] <=> l1[:consumer_version_order]} end def find_compatible_pacticipant_versions selectors - find(selectors) - .group_by{|line| GROUP_BY_PROVIDER_VERSION_NUMBER.collect{|key| line[key] }} - .values - .collect{ | lines | lines.first[:provider_version_number].nil? ? lines : lines.last } - .flatten + + find(selectors, latestby: 'cvpv') .select{|line| line[:success] } end ## # If the version is nil, it means all versions for that pacticipant are to be included # - def find_all selectors + def find_all selectors, options selectors = look_up_versions_for_tags(selectors) - query = Row.select_all + query = base_table(options).select_all if selectors.size == 1 query = where_consumer_or_provider_is(selectors.first, query) @@ -80,6 +76,11 @@ def find_all selectors .collect(&:values) end + def base_table(options) + return Row unless options[:latestby] + return LatestRow + end + def look_up_versions_for_tags(selectors) selectors.collect do | selector | # resource validation currently stops tag being specified without latest=true diff --git a/lib/pact_broker/matrix/row.rb b/lib/pact_broker/matrix/row.rb index b1a4a5923..7e72dec6f 100644 --- a/lib/pact_broker/matrix/row.rb +++ b/lib/pact_broker/matrix/row.rb @@ -8,6 +8,10 @@ class Row < Sequel::Model dataset_module do include PactBroker::Repositories::Helpers end + + def summary + "#{consumer_name}#{consumer_version_number} #{provider_name}#{provider_version_number || '?'} (r#{pact_revision_number}n#{verification_number || '?'})" + end end end end diff --git a/script/seed-matrix.rb b/script/seed-matrix.rb index 31e904e96..a0644c166 100755 --- a/script/seed-matrix.rb +++ b/script/seed-matrix.rb @@ -42,8 +42,9 @@ TestDataBuilder.new .create_pact_with_hierarchy("A", "1", "B") .create_verification(provider_version: '1', success: false) - .create_verification(provider_version: '2', number: 2) - .create_verification(provider_version: '4', number: 3) + .create_verification(provider_version: '1', number: 2, success: true) + .create_verification(provider_version: '2', number: 3) + .create_verification(provider_version: '4', number: 4) .create_provider_version("5") .use_consumer("B") .use_consumer_version("1") @@ -55,17 +56,23 @@ .create_verification(provider_version: '2', success: true) .create_consumer_version("3") .create_pact - .create_pact_with_hierarchy("the-example-application", "391c43cae8c0e83c570c191f7324fccd67e53abc", "another-example-application") - .create_verification(provider_version: '391c43cae8c0e83c570c191f7324fccd67e53abc') - .create_verification(provider_version: '57fa24e44efc4d8aa42bb855a8217f145b5b1b5b', number: 2, success: false) - .create_verification(provider_version: '4', number: 3) - .use_consumer("another-example-application") - .use_consumer_version("391c43cae8c0e83c570c191f7324fccd67e53abc") - .create_provider("a-third-example-application") + .use_consumer("A") + .create_consumer_version("2") + .use_provider("B") .create_pact - .create_verification(provider_version: '391c43cae8c0e83c570c191f7324fccd67e53abc', success: false) - .use_consumer_version("57fa24e44efc4d8aa42bb855a8217f145b5b1b5b") - .create_pact - .create_verification(provider_version: '57fa24e44efc4d8aa42bb855a8217f145b5b1b5b', success: true) + .create_verification(provider_version: '5') + + # .create_pact_with_hierarchy("the-example-application", "391c43cae8c0e83c570c191f7324fccd67e53abc", "another-example-application") + # .create_verification(provider_version: '391c43cae8c0e83c570c191f7324fccd67e53abc') + # .create_verification(provider_version: '57fa24e44efc4d8aa42bb855a8217f145b5b1b5b', number: 2, success: false) + # .create_verification(provider_version: '4', number: 3) + # .use_consumer("another-example-application") + # .use_consumer_version("391c43cae8c0e83c570c191f7324fccd67e53abc") + # .create_provider("a-third-example-application") + # .create_pact + # .create_verification(provider_version: '391c43cae8c0e83c570c191f7324fccd67e53abc', success: false) + # .use_consumer_version("57fa24e44efc4d8aa42bb855a8217f145b5b1b5b") + # .create_pact + # .create_verification(provider_version: '57fa24e44efc4d8aa42bb855a8217f145b5b1b5b', success: true) diff --git a/spec/lib/pact_broker/api/decorators/matrix_decorator_spec.rb b/spec/lib/pact_broker/api/decorators/matrix_decorator_spec.rb index 5fad64dc6..ba2a2db6c 100644 --- a/spec/lib/pact_broker/api/decorators/matrix_decorator_spec.rb +++ b/spec/lib/pact_broker/api/decorators/matrix_decorator_spec.rb @@ -18,8 +18,8 @@ module Decorators provider_version_number: "4.5.6", provider_name: "Provider", success: line_1_success, - number: 1, - build_url: nil, + verification_number: 1, + verification_build_url: nil, verification_executed_at: verification_date } end @@ -33,8 +33,8 @@ module Decorators provider_version_number: nil, provider_name: "Provider", success: line_2_success, - number: nil, - build_url: nil, + verification_number: nil, + verification_build_url: nil, verification_executed_at: verification_date } end diff --git a/spec/lib/pact_broker/matrix/repository_spec.rb b/spec/lib/pact_broker/matrix/repository_spec.rb index 416917345..f939a9fc4 100644 --- a/spec/lib/pact_broker/matrix/repository_spec.rb +++ b/spec/lib/pact_broker/matrix/repository_spec.rb @@ -12,7 +12,7 @@ def build_selectors(hash) end def shorten_row row - "#{row[:consumer_name]}#{row[:consumer_version_number]} #{row[:provider_name]}#{row[:provider_version_number] || '?'} n#{row[:number] || '?'}" + "#{row[:consumer_name]}#{row[:consumer_version_number]} #{row[:provider_name]}#{row[:provider_version_number] || '?'} n#{row[:verification_number] || '?'}" end def shorten_rows rows @@ -23,6 +23,7 @@ def shorten_rows rows before do # A1 - B1 # A1 - B1 r2 + # A1 - B2 r3 # A1 - C1 # A2 - B? # A2 - C2 @@ -46,7 +47,8 @@ def shorten_rows rows let(:a1_b1_n1) { "A1 B1 n1" } let(:a1_b1_n2) { "A1 B1 n2" } let(:a1_b2_n3) { "A1 B2 n3" } - let(:a1_c1_n1) { "A1 C1 n1"} + let(:a1_c1_n1) { "A1 C1 n1" } + let(:a2_b__n_) { "A2 B? n?" } context "when just the consumer name is specified" do let(:selectors) { build_selectors('A' => nil) } @@ -56,6 +58,7 @@ def shorten_rows rows expect(subject).to include a1_b1_n1 expect(subject).to include a1_b1_n2 expect(subject).to include a1_c1_n1 + expect(subject).to include a2_b__n_ expect(subject.size).to eq 6 end end @@ -67,6 +70,7 @@ def shorten_rows rows expect(subject).to_not include a1_b1_n1 expect(subject).to include a1_b1_n2 expect(subject).to include a1_c1_n1 + expect(subject).to include a2_b__n_ expect(subject.size).to eq 5 end end @@ -79,6 +83,7 @@ def shorten_rows rows expect(subject).to_not include a1_b1_n2 expect(subject).to include a1_b2_n3 expect(subject).to include a1_c1_n1 + expect(subject).to include a2_b__n_ expect(subject.size).to eq 4 end end @@ -554,8 +559,8 @@ def shorten_rows rows describe "#find_compatible_pacticipant_versions" do let(:td) { TestDataBuilder.new } - # subject { Repository.new.find_compatible_pacticipant_versions(selectors) } - subject { Repository.new.find(selectors, success: [true], scope: 'latest')} + + subject { Repository.new.find(selectors, success: [true], latestby: 'cvpv')} context "when compatible versions can be found" do before do @@ -582,7 +587,7 @@ def shorten_rows rows expect(subject.first[:consumer_version_number]).to eq "1" expect(subject.first[:provider_name]).to eq "B" expect(subject.first[:provider_version_number]).to eq "2" - expect(subject.first[:number]).to eq 2 + expect(subject.first[:verification_number]).to eq 2 expect(subject.first[:pact_created_at]).to be_datey expect(subject.first[:verification_executed_at]).to be_datey @@ -590,7 +595,7 @@ def shorten_rows rows expect(subject.last[:consumer_version_number]).to eq "2" expect(subject.last[:provider_name]).to eq "C" expect(subject.last[:provider_version_number]).to eq "2" - expect(subject.last[:number]).to eq 1 + expect(subject.last[:verification_number]).to eq 1 expect(subject.last[:pact_created_at]).to be_datey expect(subject.size).to eq 2 @@ -598,6 +603,7 @@ def shorten_rows rows context "when one or more pacticipants does not have a version specified" do let(:selectors){ build_selectors("A" => "1", "B" => "2", "C" => nil) } + let(:options) { { latestby: 'cvpv'} } it "returns all the rows for that pacticipant" do expect(subject).to include_hash_matching(provider_name: "C", provider_version_number: "2") @@ -628,7 +634,7 @@ def shorten_rows rows it "returns the last line" do expect(subject.size).to eq 1 - expect(subject.first[:number]).to eq 2 + expect(subject).to include_hash_matching verification_number: 2 end end diff --git a/spec/migrations/50_create_latest_matrix_spec.rb b/spec/migrations/50_create_latest_matrix_spec.rb new file mode 100644 index 000000000..d5f700d4d --- /dev/null +++ b/spec/migrations/50_create_latest_matrix_spec.rb @@ -0,0 +1,114 @@ +describe 'create latest matrix (latest pact revision/latest verification for provider version)', migration: true do + before do + PactBroker::Database.migrate(50) + end + + def shorten_row row + "#{row[:consumer_name]}#{row[:consumer_version_number]} #{row[:provider_name]}#{row[:provider_version_number] || '?'} (r#{row[:pact_revision_number]}/n#{row[:verification_number] || '?'})" + end + + let(:now) { DateTime.new(2018, 2, 2) } + let!(:consumer) { create(:pacticipants, {name: 'C', created_at: now, updated_at: now}) } + let!(:provider) { create(:pacticipants, {name: 'P', created_at: now, updated_at: now}) } + let!(:consumer_version_1) { create(:versions, {number: '1', order: 1, pacticipant_id: consumer[:id], created_at: now, updated_at: now}) } + let!(:consumer_version_2) { create(:versions, {number: '2', order: 2, pacticipant_id: consumer[:id], created_at: now, updated_at: now}) } + let!(:provider_version_1) { create(:versions, {number: '1', order: 1, pacticipant_id: provider[:id], created_at: now, updated_at: now}) } + let!(:provider_version_2) { create(:versions, {number: '2', order: 2, pacticipant_id: provider[:id], created_at: now, updated_at: now}) } + let!(:pact_version_1) { create(:pact_versions, {content: {some: 'json'}.to_json, sha: '1', consumer_id: consumer[:id], provider_id: provider[:id], created_at: now}) } + let!(:pact_version_2) { create(:pact_versions, {content: {some: 'json other'}.to_json, sha: '2', consumer_id: consumer[:id], provider_id: provider[:id], created_at: now}) } + let!(:pact_version_3) { create(:pact_versions, {content: {some: 'json more'}.to_json, sha: '3', consumer_id: consumer[:id], provider_id: provider[:id], created_at: now}) } + let!(:pact_publication_1) do + create(:pact_publications, { + consumer_version_id: consumer_version_1[:id], + provider_id: provider[:id], + revision_number: 1, + pact_version_id: pact_version_1[:id], + created_at: now + }) + end + + let!(:pact_publication_2) do + create(:pact_publications, { + consumer_version_id: consumer_version_1[:id], + provider_id: provider[:id], + revision_number: 2, + pact_version_id: pact_version_2[:id], + created_at: now + }) + end + + # C2 P? (r1/n?) + let!(:pact_publication_3) do + create(:pact_publications, { + consumer_version_id: consumer_version_2[:id], + provider_id: provider[:id], + revision_number: 1, + pact_version_id: pact_version_3[:id], + created_at: now + }) + end + + # C1 P3 (r1n3) + let!(:verification_1) do + create(:verifications, { + number: 1, + success: true, + provider_version_id: provider_version_1[:id], + pact_version_id: pact_version_1[:id], + execution_date: now, + created_at: now + }) + end + + # C1 P3 (r1n1) + let!(:verification_2) do + create(:verifications, { + number: 1, + success: true, + provider_version_id: provider_version_1[:id], + pact_version_id: pact_version_2[:id], + execution_date: now, + created_at: now + }) + end + + # include + let!(:verification_3) do + create(:verifications, { + number: 2, + success: true, + provider_version_id: provider_version_1[:id], + pact_version_id: pact_version_2[:id], + execution_date: now, + created_at: now + }) + end + + # include + let!(:verification_4) do + create(:verifications, { + number: 3, + success: true, + provider_version_id: provider_version_2[:id], + pact_version_id: pact_version_2[:id], + execution_date: now, + created_at: now + }) + end + + # C1 P1 (r1/n1) this pact version is overwritten by r2 + # C1 P1 (r2/n1) this verification is overwritten by n2 + # C1 P1 (r2/n2) + # C1 P2 (r2/n3) + # C2 P? (r1/n?) + + it "only includes the latest pact revisions and latest verifications" do + puts database[:matrix].order(:consumer_version_order, :provider_version_order, :pact_revision_number, :verification_id).all.collect{ |r| shorten_row(r) } + rows = database[:latest_matrix].order(:verification_id).all.collect{|row| shorten_row(row) } + expect(rows).to include "C1 P1 (r2/n2)" + expect(rows).to include "C1 P2 (r2/n3)" + expect(rows).to include "C2 P? (r1/n?)" + expect(rows).to_not include "C1 P1 (r2/n1)" + expect(database[:latest_matrix].count).to eq 3 + end +end diff --git a/spec/migrations/change_migration_strategy_spec.rb b/spec/migrations/change_migration_strategy_spec.rb index af75ab841..a46ca17f3 100644 --- a/spec/migrations/change_migration_strategy_spec.rb +++ b/spec/migrations/change_migration_strategy_spec.rb @@ -58,7 +58,7 @@ puts output output = `bundle exec rake pact_broker:db:version` puts output - expect(output.strip).to eq "47" + expect(output.strip.to_i).to be > 47 end end