From ac7c450fb18cfcabecf6e35389368f1d98c3c36e Mon Sep 17 00:00:00 2001 From: "nastassia.dailidava" Date: Wed, 24 Jan 2024 18:14:54 +0100 Subject: [PATCH] Implemented locality weighted load balancing (added tests) --- CHANGELOG.md | 5 + docs/configuration.md | 18 +-- .../snapshot/EnvoySnapshotFactory.kt | 14 +-- .../snapshot/SnapshotProperties.kt | 6 +- .../resource/clusters/EnvoyClustersFactory.kt | 2 +- .../endpoints/EnvoyEndpointsFactory.kt | 40 +++---- .../envoycontrol/v3/SimpleCacheTest.java | 7 +- .../envoycontrol/EnvoySnapshotFactoryTest.kt | 105 ++++++++++++++---- .../clusters/EnvoyClustersFactoryTest.kt | 6 +- .../endpoints/EnvoyEndpointsFactoryTest.kt | 52 --------- .../envoycontrol/utils/EndpointsOperations.kt | 2 +- .../envoycontrol/utils/TestData.kt | 18 ++- .../WeightedClustersRoutingTest.kt | 8 +- 13 files changed, 148 insertions(+), 135 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5bf2f31f5..4071f6718 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ Lists all changes with user impact. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). +## [0.20.10] +### Changed +- Implemented locality weighted load balancing + + ## [0.20.9] ### Changed - Configurable path normalization diff --git a/docs/configuration.md b/docs/configuration.md index b6f6d13c6..39acb31e4 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -129,14 +129,16 @@ Property **envoy-control.envoy.snapshot.outgoing-permissions.rbac.clients-lists.custom-clients-lists** | Lists of clients which will be applied to each rbac policy, only if key for defined list is present in clients for defined endpoint | empty map ## Load Balancing -Property | Description | Default value -------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------- -**envoy-control.envoy.snapshot.load-balancing.weights.enabled** | if set to true, weighted load balancing will be enabled | false -**envoy-control.envoy.snapshot.load-balancing.canary.enabled** | if set to true, routing to canary instances based on *canary header* will be enabled (corresponding Envoy static config is required, see [docs](features/load_balancing.md)) | false -**envoy-control.envoy.snapshot.load-balancing.canary.metadata-key** | metadata that will be set for canary EDS endpoints - key (must match Envoy static `header_to_metadata` filter config, see [docs](features/load_balancing.md)) | canary -**envoy-control.envoy.snapshot.load-balancing.canary.header-value** | only when *canary header* is set to this value request will be routed to canary instances (*canary header* name is set in Envoy static config, see [docs](features/load_balancing.md)) | 1 -**envoy-control.envoy.snapshot.load-balancing.policy** | load balancing policy used for clusters. [Accepted values](https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto#enum-config-cluster-v3-cluster-lbpolicy) | LEAST_REQUEST -**envoy-control.envoy.snapshot.load-balancing.use-keys-subset-fallback-policy** | KEYS_SUBSET fallback policy is used by default when canary and service-tags are enabled. It is not supported in Envoy <= 1.12.x. Set to false for compatibility with Envoy 1.12.x | true +Property | Description | Default value +------------------------------------------------------------------------------------------- |----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| --------- +**envoy-control.envoy.snapshot.load-balancing.weights.enabled** | if set to true, weighted load balancing will be enabled | false +**envoy-control.envoy.snapshot.load-balancing.canary.enabled** | if set to true, routing to canary instances based on *canary header* will be enabled (corresponding Envoy static config is required, see [docs](features/load_balancing.md)) | false +**envoy-control.envoy.snapshot.load-balancing.canary.metadata-key** | metadata that will be set for canary EDS endpoints - key (must match Envoy static `header_to_metadata` filter config, see [docs](features/load_balancing.md)) | canary +**envoy-control.envoy.snapshot.load-balancing.canary.header-value** | only when *canary header* is set to this value request will be routed to canary instances (*canary header* name is set in Envoy static config, see [docs](features/load_balancing.md)) | 1 +**envoy-control.envoy.snapshot.load-balancing.policy** | load balancing policy used for clusters. [Accepted values](https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto#enum-config-cluster-v3-cluster-lbpolicy) | LEAST_REQUEST +**envoy-control.envoy.snapshot.load-balancing.use-keys-subset-fallback-policy** | KEYS_SUBSET fallback policy is used by default when canary and service-tags are enabled. It is not supported in Envoy <= 1.12.x. Set to false for compatibility with Envoy 1.12.x | true +**envoy-control.envoy.snapshot.load-balancing.traffic-splitting.zoneName** | a zone to which traffic will be routed if traffic splitting is enabled | "" +**envoy-control.envoy.snapshot.load-balancing.traffic-splitting.weights-by-service-properties.** | a map that maps service name to a map [zoneName: weight] | empty map ## Routing diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/EnvoySnapshotFactory.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/EnvoySnapshotFactory.kt index 7e64e74c7..df4745bc4 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/EnvoySnapshotFactory.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/EnvoySnapshotFactory.kt @@ -224,7 +224,7 @@ class EnvoySnapshotFactory( globalSnapshot: GlobalSnapshot, ): RouteSpecification { val trafficSplitting = properties.loadBalancing.trafficSplitting - val weights = trafficSplitting.serviceByWeightsProperties[serviceName] + val weights = trafficSplitting.weightsByService[serviceName] val enabledForDependency = globalSnapshot.endpoints[clusterName]?.endpointsList ?.any { e -> trafficSplitting.zoneName == e.locality.zone } ?: false @@ -268,18 +268,16 @@ class EnvoySnapshotFactory( // endpointsFactory.filterEndpoints() can use this cache to prevent computing the same // ClusterLoadAssignments many times - it may reduce MEM, CPU and latency if some serviceTags are // commonly used - routeSpec.clusterName to endpointsFactory.filterEndpoints(endpoints, routeSpec.settings.routingPolicy) + endpointsFactory.filterEndpoints(endpoints, routeSpec.settings.routingPolicy).let { + endpointsFactory.assignLocalityWeights(routeSpec, it) + } } - }.toMap() + } val rateLimitClusters = if (rateLimitEndpoints.isNotEmpty()) listOf(properties.rateLimit.serviceName) else emptyList() val rateLimitLoadAssignments = rateLimitClusters.mapNotNull { name -> globalSnapshot.endpoints[name] } - val loadAssignments = endpointsFactory.assignLocalityWeights( - egressLoadAssignments, - egressRouteSpecifications - ) - return loadAssignments + rateLimitLoadAssignments + return egressLoadAssignments + rateLimitLoadAssignments } private fun newSnapshotForGroup( diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotProperties.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotProperties.kt index 49f490c68..19973e06f 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotProperties.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotProperties.kt @@ -165,13 +165,11 @@ class CanaryProperties { class TrafficSplittingProperties { var zoneName = "" var headerName = "" // todo - var serviceByWeightsProperties: Map = mapOf() + var weightsByService: Map = mapOf() } class ZoneWeights { - var main = 100 // todo remove - var secondary = 0 - var zoneByWeights: Map = mapOf() + var weightByZone: Map = mapOf() // todo convert to percents } class LoadBalancingWeightsProperties { diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactory.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactory.kt index c830085ab..f06ab030d 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactory.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactory.kt @@ -282,7 +282,7 @@ class EnvoyClustersFactory( clusterLoadAssignment: ClusterLoadAssignment? ): Boolean { val trafficSplitting = properties.loadBalancing.trafficSplitting - val trafficSplitEnabled = trafficSplitting.serviceByWeightsProperties.containsKey(serviceName) + val trafficSplitEnabled = trafficSplitting.weightsByService.containsKey(serviceName) return trafficSplitEnabled && hasEndpointsInZone(clusterLoadAssignment, trafficSplitting) } diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/endpoints/EnvoyEndpointsFactory.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/endpoints/EnvoyEndpointsFactory.kt index 4251ed2cd..19bd5e272 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/endpoints/EnvoyEndpointsFactory.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/endpoints/EnvoyEndpointsFactory.kt @@ -19,7 +19,6 @@ import pl.allegro.tech.servicemesh.envoycontrol.services.ServiceInstance import pl.allegro.tech.servicemesh.envoycontrol.services.ServiceInstances import pl.allegro.tech.servicemesh.envoycontrol.snapshot.RouteSpecification import pl.allegro.tech.servicemesh.envoycontrol.snapshot.SnapshotProperties -import pl.allegro.tech.servicemesh.envoycontrol.snapshot.StandardRouteSpecification import pl.allegro.tech.servicemesh.envoycontrol.snapshot.WeightRouteSpecification import pl.allegro.tech.servicemesh.envoycontrol.snapshot.ZoneWeights import pl.allegro.tech.servicemesh.envoycontrol.snapshot.resource.routes.ServiceTagMetadataGenerator @@ -79,36 +78,29 @@ class EnvoyEndpointsFactory( } fun assignLocalityWeights( - clusterLoadAssignments: Map, - egressRouteSpecifications: List - ): List { - val weighted = egressRouteSpecifications - .filterIsInstance() - .onEach { logger.debug("Traffic splitting is enabled for cluster: ${it.clusterName}") } - .mapNotNull { routeSpec -> - clusterLoadAssignments[routeSpec.clusterName]?.let { assignment -> - ClusterLoadAssignment.newBuilder(assignment) - .clearEndpoints() - .addAllEndpoints(assignWeights(assignment.endpointsList, routeSpec.clusterWeights)) - .setClusterName(routeSpec.clusterName) - .build() - } - } - val remaining = egressRouteSpecifications.filterIsInstance() - .mapNotNull { clusterLoadAssignments[it.clusterName] } - return (remaining + weighted) + routeSpec: RouteSpecification, + loadAssignment: ClusterLoadAssignment + ): ClusterLoadAssignment { + return if (routeSpec is WeightRouteSpecification) { + ClusterLoadAssignment.newBuilder(loadAssignment) + .clearEndpoints() + .addAllEndpoints(assignWeights(loadAssignment.endpointsList, routeSpec.clusterWeights)) + .setClusterName(routeSpec.clusterName) + .build() + } else loadAssignment } private fun assignWeights( llbEndpointsList: List, weights: ZoneWeights ): List { return llbEndpointsList - .filter { weights.zoneByWeights.containsKey(it.locality.zone) } .map { - it.toBuilder() - .setLoadBalancingWeight(UInt32Value.of(weights.zoneByWeights[it.locality.zone] ?: 0)) - .build() - }.toList() + if (weights.weightByZone.containsKey(it.locality.zone)) { + LocalityLbEndpoints.newBuilder(it) + .setLoadBalancingWeight(UInt32Value.of(weights.weightByZone[it.locality.zone] ?: 0)) + .build() + } else it + } } private fun filterEndpoints(loadAssignment: ClusterLoadAssignment, tag: String): ClusterLoadAssignment? { diff --git a/envoy-control-core/src/test/java/pl/allegro/tech/servicemesh/envoycontrol/v3/SimpleCacheTest.java b/envoy-control-core/src/test/java/pl/allegro/tech/servicemesh/envoycontrol/v3/SimpleCacheTest.java index 3c5568f71..1a70af568 100644 --- a/envoy-control-core/src/test/java/pl/allegro/tech/servicemesh/envoycontrol/v3/SimpleCacheTest.java +++ b/envoy-control-core/src/test/java/pl/allegro/tech/servicemesh/envoycontrol/v3/SimpleCacheTest.java @@ -40,7 +40,6 @@ public class SimpleCacheTest { private static final boolean ADS = ThreadLocalRandom.current().nextBoolean(); protected static final String CLUSTER_NAME = "cluster0"; - private static final String SECONDARY_CLUSTER_NAME = "cluster1"; private static final String LISTENER_NAME = "listener0"; private static final String ROUTE_NAME = "route0"; private static final String SECRET_NAME = "secret0"; @@ -65,10 +64,8 @@ public class SimpleCacheTest { VERSION2); protected static final Snapshot MULTIPLE_RESOURCES_SNAPSHOT2 = Snapshot.create( - ImmutableList.of(Cluster.newBuilder().setName(CLUSTER_NAME).build(), - Cluster.newBuilder().setName(SECONDARY_CLUSTER_NAME).build()), - ImmutableList.of(ClusterLoadAssignment.newBuilder().setClusterName(CLUSTER_NAME).build(), - ClusterLoadAssignment.newBuilder().setClusterName(SECONDARY_CLUSTER_NAME).build()), + ImmutableList.of(Cluster.newBuilder().setName(CLUSTER_NAME).build()), + ImmutableList.of(ClusterLoadAssignment.newBuilder().setClusterName(CLUSTER_NAME).build()), ImmutableList.of(Listener.newBuilder().setName(LISTENER_NAME).build()), ImmutableList.of(RouteConfiguration.newBuilder().setName(ROUTE_NAME).build()), ImmutableList.of(Secret.newBuilder().setName(SECRET_NAME).build()), diff --git a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoySnapshotFactoryTest.kt b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoySnapshotFactoryTest.kt index 77ee66946..da26a343c 100644 --- a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoySnapshotFactoryTest.kt +++ b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoySnapshotFactoryTest.kt @@ -44,15 +44,15 @@ import pl.allegro.tech.servicemesh.envoycontrol.utils.EGRESS_HOST import pl.allegro.tech.servicemesh.envoycontrol.utils.EGRESS_PORT import pl.allegro.tech.servicemesh.envoycontrol.utils.INGRESS_HOST import pl.allegro.tech.servicemesh.envoycontrol.utils.INGRESS_PORT +import pl.allegro.tech.servicemesh.envoycontrol.utils.SNAPSHOT_PROPERTIES_WITH_WEIGHTS +import pl.allegro.tech.servicemesh.envoycontrol.utils.TRAFFIC_SPLITTING_ZONE import pl.allegro.tech.servicemesh.envoycontrol.utils.createCluster import pl.allegro.tech.servicemesh.envoycontrol.utils.createClusterConfigurations import pl.allegro.tech.servicemesh.envoycontrol.utils.createEndpoints +import pl.allegro.tech.servicemesh.envoycontrol.utils.zoneWeights class EnvoySnapshotFactoryTest { companion object { - const val MAIN_CLUSTER_NAME = "service-name-2" - const val SECONDARY_CLUSTER_NAME = "service-name-2-secondary" - const val AGGREGATE_CLUSTER_NAME = "service-name-2-aggregate" const val SERVICE_NAME_2 = "service-name-2" } @@ -221,33 +221,100 @@ class EnvoySnapshotFactoryTest { assertThat(actualCluster2.commonHttpProtocolOptions.idleTimeout.seconds).isEqualTo(12) } - @Test //todo - fun `should get regular snapshot clusters when traffic splitting zone condition isn't complied`() { - // given - val defaultProperties = SnapshotProperties().also { - it.dynamicListeners.enabled = false - it.loadBalancing.trafficSplitting.serviceByWeightsProperties = mapOf( - DEFAULT_SERVICE_NAME to DEFAULT_CLUSTER_WEIGHTS - ) + @Test + fun `should get regular snapshot cluster when there are no traffic splitting settings for zone`() { + val snapshotProperties = SNAPSHOT_PROPERTIES_WITH_WEIGHTS.also { it.loadBalancing.trafficSplitting.zoneName = "not-matching-dc" } + val envoySnapshotFactory = createSnapshotFactory(snapshotProperties) + val cluster1 = createCluster(snapshotProperties, clusterName = DEFAULT_SERVICE_NAME) + val cluster2 = createCluster(snapshotProperties, clusterName = SERVICE_NAME_2) + val group: Group = createServicesGroup( + dependencies = arrayOf(SERVICE_NAME_2 to null), + snapshotProperties = snapshotProperties + ) + val globalSnapshot = createGlobalSnapshot(cluster1, cluster2) + val snapshot = envoySnapshotFactory.getSnapshotForGroup(group, globalSnapshot) + + assertThat(snapshot.clusters().resources().values) + .allSatisfy { !it.hasCommonLbConfig() || !it.commonLbConfig.hasLocalityWeightedLbConfig() } + .hasSize(1) + } + + @Test + fun `should get cluster with locality weighted config when there are traffic splitting settings for zone`() { + val envoySnapshotFactory = createSnapshotFactory(SNAPSHOT_PROPERTIES_WITH_WEIGHTS) + val cluster1 = createCluster(SNAPSHOT_PROPERTIES_WITH_WEIGHTS, clusterName = DEFAULT_SERVICE_NAME) + val cluster2 = createCluster(SNAPSHOT_PROPERTIES_WITH_WEIGHTS, clusterName = SERVICE_NAME_2) + val group: Group = createServicesGroup( + dependencies = arrayOf(SERVICE_NAME_2 to null), + snapshotProperties = SNAPSHOT_PROPERTIES_WITH_WEIGHTS + ) + val globalSnapshot = createGlobalSnapshot(cluster1, cluster2) + val snapshot = envoySnapshotFactory.getSnapshotForGroup(group, globalSnapshot) + + assertThat(snapshot.clusters().resources().values) + .anySatisfy { it.hasCommonLbConfig() && it.commonLbConfig.hasLocalityWeightedLbConfig() } + .hasSize(1) + } + + @Test + fun `should get weighted locality lb endpoints when there are traffic splitting settings for zone`() { + val envoySnapshotFactory = createSnapshotFactory(SNAPSHOT_PROPERTIES_WITH_WEIGHTS) + val cluster1 = createCluster(SNAPSHOT_PROPERTIES_WITH_WEIGHTS, clusterName = DEFAULT_SERVICE_NAME) + val cluster2 = createCluster(SNAPSHOT_PROPERTIES_WITH_WEIGHTS, clusterName = SERVICE_NAME_2) + val group: Group = createServicesGroup( + dependencies = arrayOf(SERVICE_NAME_2 to null), + snapshotProperties = SNAPSHOT_PROPERTIES_WITH_WEIGHTS + ) + val globalSnapshot = createGlobalSnapshot(cluster1, cluster2) + val snapshot = envoySnapshotFactory.getSnapshotForGroup(group, globalSnapshot) + + assertThat(snapshot.endpoints().resources().values) + .anySatisfy { + assertThat(it.endpointsList) + .anySatisfy { e -> + e.locality.zone == TRAFFIC_SPLITTING_ZONE && + e.loadBalancingWeight.value == DEFAULT_CLUSTER_WEIGHTS.weightByZone[TRAFFIC_SPLITTING_ZONE] + } + .anySatisfy { e -> + e.locality.zone == CURRENT_ZONE && + e.loadBalancingWeight.value == DEFAULT_CLUSTER_WEIGHTS.weightByZone[CURRENT_ZONE] + } + .hasSize(2) + } + } + + @Test + fun `should not set weight to locality lb endpoints when there are no matching weight settings`() { + val defaultProperties = SNAPSHOT_PROPERTIES_WITH_WEIGHTS.also { + it.loadBalancing.trafficSplitting.weightsByService = mapOf( + DEFAULT_SERVICE_NAME to zoneWeights(mapOf(CURRENT_ZONE to 60)) + ) + } val envoySnapshotFactory = createSnapshotFactory(defaultProperties) val cluster1 = createCluster(defaultProperties, clusterName = DEFAULT_SERVICE_NAME) val cluster2 = createCluster(defaultProperties, clusterName = SERVICE_NAME_2) val group: Group = createServicesGroup( - dependencies = arrayOf(cluster2.name to null), + dependencies = arrayOf(SERVICE_NAME_2 to null), snapshotProperties = defaultProperties ) val globalSnapshot = createGlobalSnapshot(cluster1, cluster2) - - // when val snapshot = envoySnapshotFactory.getSnapshotForGroup(group, globalSnapshot) - // then - assertThat(snapshot.clusters().resources()) - .containsKey(MAIN_CLUSTER_NAME) - .doesNotContainKey(SECONDARY_CLUSTER_NAME) - .doesNotContainKey(AGGREGATE_CLUSTER_NAME) + assertThat(snapshot.endpoints().resources().values) + .anySatisfy { + assertThat(it.endpointsList) + .anySatisfy { e -> + e.locality.zone == CURRENT_ZONE && + e.loadBalancingWeight.value == DEFAULT_CLUSTER_WEIGHTS.weightByZone[CURRENT_ZONE] + } + .anySatisfy { e -> + e.locality.zone == TRAFFIC_SPLITTING_ZONE && + !e.hasLoadBalancingWeight() + } + .hasSize(2) + } } @Test diff --git a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactoryTest.kt b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactoryTest.kt index fab5d0afd..9c82af855 100644 --- a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactoryTest.kt +++ b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactoryTest.kt @@ -12,7 +12,7 @@ import pl.allegro.tech.servicemesh.envoycontrol.utils.CLUSTER_NAME1 import pl.allegro.tech.servicemesh.envoycontrol.utils.CLUSTER_NAME2 import pl.allegro.tech.servicemesh.envoycontrol.utils.DEFAULT_CLUSTER_WEIGHTS import pl.allegro.tech.servicemesh.envoycontrol.utils.DEFAULT_SERVICE_NAME -import pl.allegro.tech.servicemesh.envoycontrol.utils.TRAFFIC_SPLITTING_FORCE_TRAFFIC_ZONE +import pl.allegro.tech.servicemesh.envoycontrol.utils.TRAFFIC_SPLITTING_ZONE import pl.allegro.tech.servicemesh.envoycontrol.utils.createAllServicesGroup import pl.allegro.tech.servicemesh.envoycontrol.utils.createCluster import pl.allegro.tech.servicemesh.envoycontrol.utils.createClusterConfigurations @@ -25,10 +25,10 @@ internal class EnvoyClustersFactoryTest { companion object { private val factory = EnvoyClustersFactory(SnapshotProperties()) private val snapshotPropertiesWithWeights = SnapshotProperties().apply { - loadBalancing.trafficSplitting.serviceByWeightsProperties = mapOf( + loadBalancing.trafficSplitting.weightsByService = mapOf( DEFAULT_SERVICE_NAME to DEFAULT_CLUSTER_WEIGHTS ) - loadBalancing.trafficSplitting.zoneName = TRAFFIC_SPLITTING_FORCE_TRAFFIC_ZONE + loadBalancing.trafficSplitting.zoneName = TRAFFIC_SPLITTING_ZONE } } diff --git a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/endpoints/EnvoyEndpointsFactoryTest.kt b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/endpoints/EnvoyEndpointsFactoryTest.kt index 066ef5092..eb1e693e8 100644 --- a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/endpoints/EnvoyEndpointsFactoryTest.kt +++ b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/endpoints/EnvoyEndpointsFactoryTest.kt @@ -8,7 +8,6 @@ import org.junit.jupiter.api.Test import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.Arguments import org.junit.jupiter.params.provider.MethodSource -import pl.allegro.tech.servicemesh.envoycontrol.groups.DependencySettings import pl.allegro.tech.servicemesh.envoycontrol.groups.RoutingPolicy import pl.allegro.tech.servicemesh.envoycontrol.services.ClusterState import pl.allegro.tech.servicemesh.envoycontrol.services.Locality @@ -19,12 +18,7 @@ import pl.allegro.tech.servicemesh.envoycontrol.services.ServiceName import pl.allegro.tech.servicemesh.envoycontrol.services.ServicesState import pl.allegro.tech.servicemesh.envoycontrol.snapshot.LoadBalancingPriorityProperties import pl.allegro.tech.servicemesh.envoycontrol.snapshot.LoadBalancingProperties -import pl.allegro.tech.servicemesh.envoycontrol.snapshot.RouteSpecification import pl.allegro.tech.servicemesh.envoycontrol.snapshot.SnapshotProperties -import pl.allegro.tech.servicemesh.envoycontrol.snapshot.TrafficSplittingProperties -import pl.allegro.tech.servicemesh.envoycontrol.snapshot.WeightRouteSpecification -import pl.allegro.tech.servicemesh.envoycontrol.snapshot.ZoneWeights -import pl.allegro.tech.servicemesh.envoycontrol.utils.zoneWeights import java.util.concurrent.ConcurrentHashMap import java.util.stream.Stream @@ -57,10 +51,6 @@ internal class EnvoyEndpointsFactoryTest { private val serviceName = "service-one" - private val serviceName2 = "service-two" - - private val defaultWeights = zoneWeights(50, 50) - private val defaultZone = "DC1" private val endpointsFactory = EnvoyEndpointsFactory( @@ -364,33 +354,6 @@ internal class EnvoyEndpointsFactoryTest { ) } - @Test - fun `should get empty secondary cluster endpoints for route spec with no such cluster`() { - val multiClusterState = MultiClusterState( - listOf( - clusterState(cluster = defaultZone), - clusterState(cluster = defaultZone, serviceName = serviceName2), - ) - ) - val services = setOf(serviceName, serviceName2) - val envoyEndpointsFactory = EnvoyEndpointsFactory( - snapshotPropertiesWithTrafficSplitting( - mapOf(serviceName to defaultWeights) - ), - currentZone = defaultZone - ) - val loadAssignments = envoyEndpointsFactory.createLoadAssignment( - services, - multiClusterState - ).associateBy { it.clusterName } - - val result = envoyEndpointsFactory.assignLocalityWeights( - loadAssignments, - listOf("some-other-service-name".toRouteSpecification()) - ) - assertThat(result).isEmpty() - } - private fun List.assertHasLoadAssignment(map: Map) { assertThat(this) .isNotEmpty() @@ -448,21 +411,6 @@ internal class EnvoyEndpointsFactoryTest { } } - private fun snapshotPropertiesWithTrafficSplitting( - serviceByWeights: Map, - zone: String = defaultZone - ) = - SnapshotProperties().apply { - loadBalancing.trafficSplitting = TrafficSplittingProperties().apply { - zoneName = zone - serviceByWeightsProperties = serviceByWeights - } - } - - private fun String.toRouteSpecification(weights: ZoneWeights = defaultWeights): RouteSpecification { - return WeightRouteSpecification(this, listOf(), DependencySettings(), weights) - } - private fun String.toClusterLoadAssignment(): ClusterLoadAssignment = ClusterLoadAssignment.newBuilder() .also { builder -> JsonFormat.parser().merge(this, builder) } .build() diff --git a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/utils/EndpointsOperations.kt b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/utils/EndpointsOperations.kt index a47cf5fb6..38eaf4af5 100644 --- a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/utils/EndpointsOperations.kt +++ b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/utils/EndpointsOperations.kt @@ -17,7 +17,7 @@ fun createLoadAssignments(clusters: List): List fun createEndpoints(): List = listOf( createEndpoint(CURRENT_ZONE), - createEndpoint(TRAFFIC_SPLITTING_FORCE_TRAFFIC_ZONE) + createEndpoint(TRAFFIC_SPLITTING_ZONE) ) fun createEndpoint(zone: String): LocalityLbEndpoints { diff --git a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/utils/TestData.kt b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/utils/TestData.kt index fcb486d1b..4dd10f126 100644 --- a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/utils/TestData.kt +++ b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/utils/TestData.kt @@ -1,5 +1,6 @@ package pl.allegro.tech.servicemesh.envoycontrol.utils +import pl.allegro.tech.servicemesh.envoycontrol.snapshot.SnapshotProperties import pl.allegro.tech.servicemesh.envoycontrol.snapshot.ZoneWeights const val INGRESS_HOST = "ingress-host" @@ -12,12 +13,19 @@ const val DEFAULT_DISCOVERY_SERVICE_NAME = "discovery-service-name" const val CLUSTER_NAME = "cluster-name" const val CLUSTER_NAME1 = "cluster-1" const val CLUSTER_NAME2 = "cluster-2" -const val TRAFFIC_SPLITTING_FORCE_TRAFFIC_ZONE = "dc2" +const val TRAFFIC_SPLITTING_ZONE = "dc2" const val CURRENT_ZONE = "dc1" -val DEFAULT_CLUSTER_WEIGHTS = zoneWeights(50, 50) +val DEFAULT_CLUSTER_WEIGHTS = zoneWeights(mapOf(CURRENT_ZONE to 60, TRAFFIC_SPLITTING_ZONE to 40)) -fun zoneWeights(main: Int, secondary: Int) = ZoneWeights().also { - it.main = main - it.secondary = secondary +val SNAPSHOT_PROPERTIES_WITH_WEIGHTS = SnapshotProperties().also { + it.dynamicListeners.enabled = false + it.loadBalancing.trafficSplitting.weightsByService = mapOf( + DEFAULT_SERVICE_NAME to DEFAULT_CLUSTER_WEIGHTS + ) + it.loadBalancing.trafficSplitting.zoneName = TRAFFIC_SPLITTING_ZONE +} + +fun zoneWeights(weightByZone: Map) = ZoneWeights().also { + it.weightByZone = weightByZone } diff --git a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/WeightedClustersRoutingTest.kt b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/WeightedClustersRoutingTest.kt index a2fbb1f7b..a3b416bc8 100644 --- a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/WeightedClustersRoutingTest.kt +++ b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/WeightedClustersRoutingTest.kt @@ -25,11 +25,9 @@ class WeightedClustersRoutingTest { "envoy-control.envoy.snapshot.stateSampleDuration" to Duration.ofSeconds(0), "envoy-control.sync.enabled" to true, "envoy-control.envoy.snapshot.load-balancing.trafficSplitting.zoneName" to forceTrafficZone, - "envoy-control.envoy.snapshot.load-balancing.trafficSplitting.serviceByWeightsProperties.$serviceName.main" to 90, - "envoy-control.envoy.snapshot.load-balancing.trafficSplitting.serviceByWeightsProperties.$serviceName.secondary" to 10, - "envoy-control.envoy.snapshot.load-balancing.trafficSplitting.serviceByWeightsProperties.$serviceName.zoneByWeights.dc1" to 30, - "envoy-control.envoy.snapshot.load-balancing.trafficSplitting.serviceByWeightsProperties.$serviceName.zoneByWeights.dc2" to 10, - "envoy-control.envoy.snapshot.load-balancing.trafficSplitting.serviceByWeightsProperties.$serviceName.zoneByWeights.dc3" to 1, + "envoy-control.envoy.snapshot.load-balancing.trafficSplitting.weights-by-service-properties.$serviceName.weightByZone.dc1" to 30, + "envoy-control.envoy.snapshot.load-balancing.trafficSplitting.weights-by-service-properties.$serviceName.weightByZone.dc2" to 10, + "envoy-control.envoy.snapshot.load-balancing.trafficSplitting.weights-by-service-properties.$serviceName.weightByZone.dc3" to 1, "envoy-control.envoy.snapshot.load-balancing.priorities.zonePriorities" to mapOf( "dc1" to mapOf( "dc1" to 0,