From 56e6310c60a0bf5b9cc5f3a4a35f90af965fe492 Mon Sep 17 00:00:00 2001 From: Maksim Strebkov <257byte@gmail.com> Date: Wed, 21 Aug 2024 12:12:40 +0300 Subject: [PATCH] Add more filters for /voting endpoints --- Tzkt.Api/Controllers/VotingController.cs | 20 ++- .../ModelBindingContextExtension.cs | 60 +++++++ .../Parameters/Binders/EpochStatusBinder.cs | 44 +++++ Tzkt.Api/Parameters/EpochStatusParameter.cs | 71 +++++++++ Tzkt.Api/Repositories/Enums/EpochStatuses.cs | 9 ++ Tzkt.Api/Repositories/VotingRepository.cs | 150 ++++++++++-------- Tzkt.Api/Utils/SqlBuilder.cs | 19 +++ .../ContractMetadata/ContractMetadata.cs | 2 +- .../Services/TokenMetadata/TokenMetadata.cs | 6 +- .../TokenMetadata/TokenMetadataState.cs | 6 +- 10 files changed, 306 insertions(+), 81 deletions(-) create mode 100644 Tzkt.Api/Parameters/Binders/EpochStatusBinder.cs create mode 100644 Tzkt.Api/Parameters/EpochStatusParameter.cs diff --git a/Tzkt.Api/Controllers/VotingController.cs b/Tzkt.Api/Controllers/VotingController.cs index b9031cd44..68c697269 100644 --- a/Tzkt.Api/Controllers/VotingController.cs +++ b/Tzkt.Api/Controllers/VotingController.cs @@ -109,6 +109,7 @@ public Task GetProposalByHash([Required][ProtocolHash] string hash) /// /// Filter by level of the first block of the period. /// Filter by level of the last block of the period. + /// Filters by voting epoch /// Specify comma-separated list of fields to include into response or leave it undefined to return full object. If you select single field, response will be an array of values in both `.fields` and `.values` modes. /// Sorts voting periods by specified field. Supported fields: `id` (default). /// Specifies which or how many items should be skipped @@ -118,6 +119,7 @@ public Task GetProposalByHash([Required][ProtocolHash] string hash) public async Task>> GetPeriods( Int32Parameter firstLevel, Int32Parameter lastLevel, + Int32Parameter epoch, SelectParameter select, SortParameter sort, OffsetParameter offset, @@ -129,25 +131,25 @@ public async Task>> GetPeriods( #endregion if (select == null) - return Ok(await Voting.GetPeriods(firstLevel, lastLevel, sort, offset, limit)); + return Ok(await Voting.GetPeriods(firstLevel, lastLevel, epoch, sort, offset, limit)); if (select.Values != null) { if (select.Values.Length == 1) - return Ok(await Voting.GetPeriods(firstLevel, lastLevel, sort, offset, limit, select.Values[0])); + return Ok(await Voting.GetPeriods(firstLevel, lastLevel, epoch, sort, offset, limit, select.Values[0])); else - return Ok(await Voting.GetPeriods(firstLevel, lastLevel, sort, offset, limit, select.Values)); + return Ok(await Voting.GetPeriods(firstLevel, lastLevel, epoch, sort, offset, limit, select.Values)); } else { if (select.Fields.Length == 1) - return Ok(await Voting.GetPeriods(firstLevel, lastLevel, sort, offset, limit, select.Fields[0])); + return Ok(await Voting.GetPeriods(firstLevel, lastLevel, epoch, sort, offset, limit, select.Fields[0])); else { return Ok(new SelectionResponse { Cols = select.Fields, - Rows = await Voting.GetPeriods(firstLevel, lastLevel, sort, offset, limit, select.Fields) + Rows = await Voting.GetPeriods(firstLevel, lastLevel, epoch, sort, offset, limit, select.Fields) }); } } @@ -271,22 +273,24 @@ public Task GetPeriodVoter([Required][TzAddress] string address) /// /// Returns a list of voting epochs. /// - /// Sorts voting epochs by specified field. Supported fields: `id` (default). + /// Filter by voting epoch status (`no_proposals`, `voting`, `completed`, `failed`). + /// Sorts voting epochs by specified field. Supported fields: `index` (default). /// Specifies which or how many items should be skipped /// Maximum number of items to return /// [HttpGet("epochs")] public async Task>> GetEpochs( + EpochStatusParameter status, SortParameter sort, OffsetParameter offset, [Range(0, 10000)] int limit = 100) { #region validate - if (sort != null && !sort.Validate("id")) + if (sort != null && !sort.Validate("id", "index")) return new BadRequest($"{nameof(sort)}", "Sorting by the specified field is not allowed."); #endregion - return Ok(await Voting.GetEpochs(sort, offset, limit)); + return Ok(await Voting.GetEpochs(status, sort, offset, limit)); } /// diff --git a/Tzkt.Api/Extensions/ModelBindingContextExtension.cs b/Tzkt.Api/Extensions/ModelBindingContextExtension.cs index a61c03ee7..d3e03edd2 100644 --- a/Tzkt.Api/Extensions/ModelBindingContextExtension.cs +++ b/Tzkt.Api/Extensions/ModelBindingContextExtension.cs @@ -878,6 +878,66 @@ public static bool TryGetSrc1HashList(this ModelBindingContext bindingContext, s return true; } + public static bool TryGetEpochStatus(this ModelBindingContext bindingContext, string name, ref bool hasValue, out string result) + { + result = null; + var valueObject = bindingContext.ValueProvider.GetValue(name); + + if (valueObject != ValueProviderResult.None) + { + bindingContext.ModelState.SetModelValue(name, valueObject); + if (!string.IsNullOrEmpty(valueObject.FirstValue)) + { + if (!EpochStatuses.IsValid(valueObject.FirstValue)) + { + bindingContext.ModelState.TryAddModelError(name, "Invalid epoch status."); + return false; + } + hasValue = true; + result = valueObject.FirstValue; + } + } + + return true; + } + + public static bool TryGetEpochStatusList(this ModelBindingContext bindingContext, string name, ref bool hasValue, out List result) + { + result = null; + var valueObject = bindingContext.ValueProvider.GetValue(name); + + if (valueObject != ValueProviderResult.None) + { + bindingContext.ModelState.SetModelValue(name, valueObject); + if (!string.IsNullOrEmpty(valueObject.FirstValue)) + { + var rawValues = valueObject.FirstValue.Split(',', StringSplitOptions.RemoveEmptyEntries); + + if (rawValues.Length == 0) + { + bindingContext.ModelState.TryAddModelError(name, "List should contain at least one item."); + return false; + } + + hasValue = true; + result = new List(rawValues.Length); + + foreach (var rawValue in rawValues) + { + if (!EpochStatuses.IsValid(rawValue)) + { + bindingContext.ModelState.TryAddModelError(name, "List contains invalid epoch status."); + return false; + } + hasValue = true; + result.Add(rawValue); + } + } + } + + return true; + } + public static bool TryGetContractKind(this ModelBindingContext bindingContext, string name, ref bool hasValue, out int? result) { result = null; diff --git a/Tzkt.Api/Parameters/Binders/EpochStatusBinder.cs b/Tzkt.Api/Parameters/Binders/EpochStatusBinder.cs new file mode 100644 index 000000000..7910e43fe --- /dev/null +++ b/Tzkt.Api/Parameters/Binders/EpochStatusBinder.cs @@ -0,0 +1,44 @@ +using Microsoft.AspNetCore.Mvc.ModelBinding; + +namespace Tzkt.Api +{ + public class EpochStatusBinder : IModelBinder + { + public Task BindModelAsync(ModelBindingContext bindingContext) + { + var model = bindingContext.ModelName; + var hasValue = false; + + if (!bindingContext.TryGetEpochStatus($"{model}", ref hasValue, out var value)) + return Task.CompletedTask; + + if (!bindingContext.TryGetEpochStatus($"{model}.eq", ref hasValue, out var eq)) + return Task.CompletedTask; + + if (!bindingContext.TryGetEpochStatus($"{model}.ne", ref hasValue, out var ne)) + return Task.CompletedTask; + + if (!bindingContext.TryGetEpochStatusList($"{model}.in", ref hasValue, out var @in)) + return Task.CompletedTask; + + if (!bindingContext.TryGetEpochStatusList($"{model}.ni", ref hasValue, out var ni)) + return Task.CompletedTask; + + if (!hasValue) + { + bindingContext.Result = ModelBindingResult.Success(null); + return Task.CompletedTask; + } + + bindingContext.Result = ModelBindingResult.Success(new EpochStatusParameter + { + Eq = value ?? eq, + Ne = ne, + In = @in, + Ni = ni + }); + + return Task.CompletedTask; + } + } +} diff --git a/Tzkt.Api/Parameters/EpochStatusParameter.cs b/Tzkt.Api/Parameters/EpochStatusParameter.cs new file mode 100644 index 000000000..154ba2401 --- /dev/null +++ b/Tzkt.Api/Parameters/EpochStatusParameter.cs @@ -0,0 +1,71 @@ +using System.Text; +using Microsoft.AspNetCore.Mvc; +using NJsonSchema.Annotations; + +namespace Tzkt.Api +{ + [ModelBinder(BinderType = typeof(EpochStatusBinder))] + [JsonSchemaExtensionData("x-tzkt-extension", "query-parameter")] + [JsonSchemaExtensionData("x-tzkt-query-parameter", "no_proposals,voting,completed,failed")] + public class EpochStatusParameter : INormalizable + { + /// + /// **Equal** filter mode (optional, i.e. `param.eq=123` is the same as `param=123`). \ + /// Specify an epoch status to get items where the specified field is equal to the specified value. + /// + /// Example: `?status=completed`. + /// + public string Eq { get; set; } + + /// + /// **Not equal** filter mode. \ + /// Specify an epoch status to get items where the specified field is not equal to the specified value. + /// + /// Example: `?status.ne=no_proposals`. + /// + public string Ne { get; set; } + + /// + /// **In list** (any of) filter mode. \ + /// Specify a comma-separated list of epoch statuses to get items where the specified field is equal to one of the specified values. + /// + /// Example: `?status.in=completed,failed`. + /// + public List In { get; set; } + + /// + /// **Not in list** (none of) filter mode. \ + /// Specify a comma-separated list of epoch statuses to get items where the specified field is not equal to all the specified values. + /// + /// Example: `?status.ni=completed,failed`. + /// + public List Ni { get; set; } + + public string Normalize(string name) + { + var sb = new StringBuilder(); + + if (Eq != null) + { + sb.Append($"{name}.eq={Eq}&"); + } + + if (Ne != null) + { + sb.Append($"{name}.ne={Ne}&"); + } + + if (In?.Count > 0) + { + sb.Append($"{name}.in={string.Join(",", In.OrderBy(x => x))}&"); + } + + if (Ni?.Count > 0) + { + sb.Append($"{name}.ni={string.Join(",", Ni.OrderBy(x => x))}&"); + } + + return sb.ToString(); + } + } +} diff --git a/Tzkt.Api/Repositories/Enums/EpochStatuses.cs b/Tzkt.Api/Repositories/Enums/EpochStatuses.cs index 0cec3f012..6051cf214 100644 --- a/Tzkt.Api/Repositories/Enums/EpochStatuses.cs +++ b/Tzkt.Api/Repositories/Enums/EpochStatuses.cs @@ -6,5 +6,14 @@ static class EpochStatuses public const string Voting = "voting"; public const string Completed = "completed"; public const string Failed = "failed"; + + public static bool IsValid(string value) => value switch + { + NoProposals => true, + Voting => true, + Completed => true, + Failed => true, + _ => false + }; } } diff --git a/Tzkt.Api/Repositories/VotingRepository.cs b/Tzkt.Api/Repositories/VotingRepository.cs index 158eec29d..7ef659fb0 100644 --- a/Tzkt.Api/Repositories/VotingRepository.cs +++ b/Tzkt.Api/Repositories/VotingRepository.cs @@ -308,11 +308,18 @@ public async Task GetPeriod(int index) }; } - public async Task> GetPeriods(Int32Parameter firstLevel, Int32Parameter lastLevel, SortParameter sort, OffsetParameter offset, int limit) + public async Task> GetPeriods( + Int32Parameter firstLevel, + Int32Parameter lastLevel, + Int32Parameter epoch, + SortParameter sort, + OffsetParameter offset, + int limit) { var sql = new SqlBuilder(@"SELECT * FROM ""VotingPeriods""") .Filter("FirstLevel", firstLevel) .Filter("LastLevel", lastLevel) + .Filter("Epoch", epoch) .Take(sort, offset, limit, x => ("Id", "Id")); await using var db = await DataSource.OpenConnectionAsync(); @@ -346,7 +353,14 @@ public async Task> GetPeriods(Int32Parameter firstLeve }); } - public async Task GetPeriods(Int32Parameter firstLevel, Int32Parameter lastLevel, SortParameter sort, OffsetParameter offset, int limit, string[] fields) + public async Task GetPeriods( + Int32Parameter firstLevel, + Int32Parameter lastLevel, + Int32Parameter epoch, + SortParameter sort, + OffsetParameter offset, + int limit, + string[] fields) { var columns = new HashSet(fields.Length); @@ -386,6 +400,7 @@ public async Task GetPeriods(Int32Parameter firstLevel, Int32Paramet var sql = new SqlBuilder($@"SELECT {string.Join(',', columns)} FROM ""VotingPeriods""") .Filter("FirstLevel", firstLevel) .Filter("LastLevel", lastLevel) + .Filter("Epoch", epoch) .Take(sort, offset, limit, x => ("Id", "Id")); await using var db = await DataSource.OpenConnectionAsync(); @@ -497,7 +512,14 @@ public async Task GetPeriods(Int32Parameter firstLevel, Int32Paramet return result; } - public async Task GetPeriods(Int32Parameter firstLevel, Int32Parameter lastLevel, SortParameter sort, OffsetParameter offset, int limit, string field) + public async Task GetPeriods( + Int32Parameter firstLevel, + Int32Parameter lastLevel, + Int32Parameter epoch, + SortParameter sort, + OffsetParameter offset, + int limit, + string field) { var columns = new HashSet(1); @@ -534,6 +556,7 @@ public async Task GetPeriods(Int32Parameter firstLevel, Int32Parameter var sql = new SqlBuilder($@"SELECT {string.Join(',', columns)} FROM ""VotingPeriods""") .Filter("FirstLevel", firstLevel) .Filter("LastLevel", lastLevel) + .Filter("Epoch", epoch) .Take(sort, offset, limit, x => ("Id", "Id")); await using var db = await DataSource.OpenConnectionAsync(); @@ -747,79 +770,76 @@ public async Task GetEpoch(int index) }; } - public async Task> GetEpochs(SortParameter sort, OffsetParameter offset, int limit) + public async Task> GetEpochs(EpochStatusParameter status, SortParameter sort, OffsetParameter offset, int limit) { - sort ??= new SortParameter { Asc = "id" }; - var sql = new SqlBuilder(@"SELECT DISTINCT ON (""Epoch"") ""Epoch"" AS epoch FROM ""VotingPeriods""") - .Take(sort, offset, limit, x => ("Epoch", "Epoch")); - - var query = $@" - SELECT periods.* FROM ( - {sql.Query} - ) as epochs - LEFT JOIN LATERAL ( - SELECT * - FROM ""VotingPeriods"" - WHERE ""Epoch"" = epoch - ) as periods - ON true - ORDER BY ""Id""{(sort.Desc != null ? " DESC" : "")}"; + var sql = new SqlBuilder($""" + WITH + groups AS ( + SELECT "Epoch", MIN("Index") AS "FirstPeriod", MAX("Index") AS "LastPeriod" + FROM "VotingPeriods" + GROUP BY "Epoch" + ), + epochs AS ( + SELECT + g."Epoch" as "Id", + g."Epoch" as "Index", + fp."FirstLevel", + lp."LastLevel", + CASE + WHEN fp."Status" = {(int)Data.Models.PeriodStatus.NoProposals} THEN '{EpochStatuses.NoProposals}' + WHEN lp."Status" = {(int)Data.Models.PeriodStatus.Active} THEN '{EpochStatuses.Voting}' + WHEN lp."Status" = {(int)Data.Models.PeriodStatus.Success} THEN '{EpochStatuses.Completed}' + ELSE '{EpochStatuses.Failed}' + END as "Status" + FROM groups as g + INNER JOIN "VotingPeriods" as fp ON fp."Index" = g."FirstPeriod" + INNER JOIN "VotingPeriods" as lp ON lp."Index" = g."LastPeriod" + ) + SELECT * + FROM epochs + """) + .FilterA(@"""Status""", status) + .Take(sort, offset, limit, x => ("Index", "Index")); await using var db = await DataSource.OpenConnectionAsync(); - var rows = await db.QueryAsync(query, sql.Params); + var rows = await db.QueryAsync(sql.Query, sql.Params); if (!rows.Any()) return Enumerable.Empty(); - var epochs = rows.Select(x => (int)x.Epoch).ToHashSet(); + var epochs = rows.Select(x => (int)x.Index).ToList(); + + var periods = (await GetPeriods( + firstLevel: null, + lastLevel: null, + epoch: new Int32Parameter { In = epochs }, + sort: new SortParameter { Asc = "id" }, + offset: null, + limit: limit * 10)) + .GroupBy(x => x.Epoch) + .ToDictionary(k => k.Key, v => v.ToList()); + var proposals = (await GetProposals( hash: null, - new Int32Parameter { In = epochs.ToList() }, - new SortParameter { Desc = "votingPower" }, - null, limit * 10)) + epoch: new Int32Parameter { In = epochs }, + sort: new SortParameter { Desc = "votingPower" }, + offset: null, + limit: limit * 10)) .GroupBy(x => x.Epoch) .ToDictionary(k => k.Key, v => v.ToList()); - return rows - .GroupBy(x => x.Epoch) - .Select(group => + return rows.Select(row => + { + return new VotingEpoch { - var periods = group.OrderBy(x => x.Index); - return new VotingEpoch - { - Index = group.Key, - FirstLevel = periods.First().FirstLevel, - StartTime = Time[periods.First().FirstLevel], - LastLevel = periods.Last().LastLevel, - EndTime = Time[periods.Last().LastLevel], - Status = GetEpochStatus(periods), - Periods = periods.Select(row => new VotingPeriod - { - Index = row.Index, - Epoch = row.Epoch, - FirstLevel = row.FirstLevel, - StartTime = Time[row.FirstLevel], - LastLevel = row.LastLevel, - EndTime = Time[row.LastLevel], - Kind = PeriodKinds.ToString(row.Kind), - Status = PeriodStatuses.ToString(row.Status), - Dictator = PeriodDictatorStatuses.ToString(row.Dictator), - TotalBakers = row.TotalBakers, - TotalVotingPower = row.TotalVotingPower, - UpvotesQuorum = row.UpvotesQuorum == null ? null : row.UpvotesQuorum / 100.0, - ProposalsCount = row.ProposalsCount, - TopUpvotes = row.TopUpvotes, - TopVotingPower = row.TopVotingPower, - BallotsQuorum = row.BallotsQuorum == null ? null : row.BallotsQuorum / 100.0, - Supermajority = row.Supermajority == null ? null : row.Supermajority / 100.0, - YayBallots = row.YayBallots, - YayVotingPower = row.YayVotingPower, - NayBallots = row.NayBallots, - NayVotingPower = row.NayVotingPower, - PassBallots = row.PassBallots, - PassVotingPower = row.PassVotingPower, - }), - Proposals = proposals.GetValueOrDefault((int)group.Key) ?? Enumerable.Empty() - }; - }); + Index = row.Index, + FirstLevel = row.FirstLevel, + StartTime = Time[row.FirstLevel], + LastLevel = row.LastLevel, + EndTime = Time[row.LastLevel], + Status = row.Status, + Periods = periods.GetValueOrDefault((int)row.Index) ?? Enumerable.Empty(), + Proposals = proposals.GetValueOrDefault((int)row.Index) ?? Enumerable.Empty() + }; + }); } public async Task GetLatestVoting() diff --git a/Tzkt.Api/Utils/SqlBuilder.cs b/Tzkt.Api/Utils/SqlBuilder.cs index 4400723c2..a775e0e3f 100644 --- a/Tzkt.Api/Utils/SqlBuilder.cs +++ b/Tzkt.Api/Utils/SqlBuilder.cs @@ -210,6 +210,25 @@ public SqlBuilder FilterA(string column, ContractKindParameter kind) return this; } + public SqlBuilder FilterA(string column, EpochStatusParameter status) + { + if (status == null) return this; + + if (status.Eq != null) + AppendFilter($"{column} = {Param(status.Eq)}"); + + if (status.Ne != null) + AppendFilter($"{column} != {Param(status.Ne)}"); + + if (status.In != null) + AppendFilter($"{column} = ANY ({Param(status.In)})"); + + if (status.Ni != null && status.Ni.Count > 0) + AppendFilter($"NOT ({column} = ANY ({Param(status.Ni)}))"); + + return this; + } + public SqlBuilder FilterA(string column, StakingActionParameter action) { if (action == null) return this; diff --git a/Tzkt.Sync/Services/ContractMetadata/ContractMetadata.cs b/Tzkt.Sync/Services/ContractMetadata/ContractMetadata.cs index a4a53f984..46f45bb89 100644 --- a/Tzkt.Sync/Services/ContractMetadata/ContractMetadata.cs +++ b/Tzkt.Sync/Services/ContractMetadata/ContractMetadata.cs @@ -218,7 +218,7 @@ class DipDupQuery class DipDupItem { [JsonPropertyName("update_id")] - public int UpdateId { get; set; } + public long UpdateId { get; set; } [JsonPropertyName("contract")] public string Contract { get; set; } diff --git a/Tzkt.Sync/Services/TokenMetadata/TokenMetadata.cs b/Tzkt.Sync/Services/TokenMetadata/TokenMetadata.cs index d20c5bdf2..800be18d7 100644 --- a/Tzkt.Sync/Services/TokenMetadata/TokenMetadata.cs +++ b/Tzkt.Sync/Services/TokenMetadata/TokenMetadata.cs @@ -303,7 +303,7 @@ async Task GetDipDupSentinel(DipDupConfig dipDupConfig) return items[0].CreatedAt; } - async Task> GetDipDupMetadata(int lastUpdateId, DipDupConfig dipDupConfig) + async Task> GetDipDupMetadata(long lastUpdateId, DipDupConfig dipDupConfig) { using var client = new HttpClient(); using var res = (await client.PostAsync(dipDupConfig.Url, new StringContent( @@ -336,7 +336,7 @@ async Task> GetDipDupMetadata(Dictionary<(string, string), T NumberHandling = JsonNumberHandling.AllowReadingFromString, MaxDepth = 10240 }; - var lastUpdateId = -1; + var lastUpdateId = -1L; while (true) { using var res = (await client.PostAsync(dipDupConfig.Url, new StringContent( @@ -408,7 +408,7 @@ class DipDupQuery class DipDupItem { [JsonPropertyName("update_id")] - public int UpdateId { get; set; } + public long UpdateId { get; set; } [JsonPropertyName("contract")] public string Contract { get; set; } diff --git a/Tzkt.Sync/Services/TokenMetadata/TokenMetadataState.cs b/Tzkt.Sync/Services/TokenMetadata/TokenMetadataState.cs index 0ddccd6d6..69d3e36a7 100644 --- a/Tzkt.Sync/Services/TokenMetadata/TokenMetadataState.cs +++ b/Tzkt.Sync/Services/TokenMetadata/TokenMetadataState.cs @@ -1,6 +1,4 @@ -using System.Collections.Generic; - -namespace Tzkt.Sync.Services +namespace Tzkt.Sync.Services { public class TokenMetadataState { @@ -9,7 +7,7 @@ public class TokenMetadataState public class DipDupState { - public int LastUpdateId { get; set; } = 0; + public long LastUpdateId { get; set; } = 0; public long LastTokenId { get; set; } = 0; // TzKT internal ID public int LastIndexedAt { get; set; } = 0; // TzKT internals public long LastIndexedAtId { get; set; } = 0; // TzKT internals