Skip to content

Commit

Permalink
Merge pull request #38 from statsig-io/exposure-tests
Browse files Browse the repository at this point in the history 
add exposure e2e tests and fix bugs
  • Loading branch information
@jkw-statsig
jkw-statsig authored Sep 22, 2021
2 parents 9f0960d + 96cf484 commit 239aec6
Show file tree
Hide file tree
Showing 4 changed files with 93 additions and 17 deletions.
81 changes: 73 additions & 8 deletions dotnet-statsig-tests/Server/ServerSDKConsistencyTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ public class ServerSDKConsistencyTest : IAsyncLifetime
private class TestData
{
public StatsigUser user { get; set; }
public Dictionary<string, bool> feature_gates { get; set; }
public Dictionary<string, FeatureGate> feature_gates_v2 { get; set; }
public Dictionary<string, DynamicConfig> dynamic_configs { get; set; }
}

Expand All @@ -41,7 +41,7 @@ public async Task InitializeAsync()
}
}

public async Task DisposeAsync() {}
public async Task DisposeAsync() { }

[Fact]
public async void TestProd()
Expand Down Expand Up @@ -73,6 +73,12 @@ public async void TestUSEast()
await TestConsistency("https://us-east-2.api.statsig.com/v1");
}

[Fact]
public async void TestEU()
{
await TestConsistency("https://az-northeurope.api.statsig.com/v1");
}

private async Task<TestData[]> FetchTestData(string apiURLBase)
{
using (HttpClient client = new HttpClient())
Expand Down Expand Up @@ -102,21 +108,80 @@ private async Task TestConsistency(string apiURLBase)
var testData = await FetchTestData(apiURLBase);
foreach (var data in testData)
{
foreach (var gate in data.feature_gates)
foreach (var gate in data.feature_gates_v2)
{
var sdkValue = await driver.CheckGate(data.user, gate.Key);
Assert.True(sdkValue == gate.Value, gate.Key + " expected " + gate.Value + " got " + sdkValue + "for " + gate.Key);
var sdkResult = driver.evaluator.CheckGate(data.user, gate.Key).GateValue;
var serverResult = gate.Value;
Assert.True(sdkResult.Value == serverResult.Value, string.Format("Values are different for gate {0}. Expected {1} but got {2}", gate.Key, serverResult.Value, sdkResult.Value));
Assert.True(sdkResult.RuleID == serverResult.RuleID, string.Format("Rule IDs are different for gate {0}. Expected {1} but got {2}", gate.Key, serverResult.RuleID, sdkResult.RuleID));
Assert.True(compareSecondaryExposures(sdkResult.SecondaryExposures, serverResult.SecondaryExposures),
string.Format("Secondary exposures are different for gate {0}. Expected {1} but got {2}", gate.Key, stringifyExposures(serverResult.SecondaryExposures), stringifyExposures(sdkResult.SecondaryExposures)));
}
foreach (var config in data.dynamic_configs)
{
var sdkValue = await driver.GetConfig(data.user, config.Key);
foreach (var entry in sdkValue.Value)
var sdkResult = driver.evaluator.GetConfig(data.user, config.Key).ConfigValue;
var serverResult = config.Value;
foreach (var entry in sdkResult.Value)
{
Assert.True(JToken.DeepEquals(entry.Value, config.Value.Value[entry.Key]));
Assert.True(JToken.DeepEquals(entry.Value, serverResult.Value[entry.Key]),
string.Format("Values are different for config {0}.", config.Key));
}
Assert.True(sdkResult.RuleID == serverResult.RuleID, string.Format("Rule IDs are different for config {0}. Expected {1} but got {2}", config.Key, serverResult.RuleID, sdkResult.RuleID));
Assert.True(compareSecondaryExposures(sdkResult.SecondaryExposures, serverResult.SecondaryExposures),
string.Format("Secondary exposures are different for config {0}. Expected {1} but got {2}", config.Key, stringifyExposures(serverResult.SecondaryExposures), stringifyExposures(sdkResult.SecondaryExposures)));
}
}
driver.Shutdown();
}

private bool compareSecondaryExposures(List<IReadOnlyDictionary<string, string>> exposures1, List<IReadOnlyDictionary<string, string>> exposures2)
{
if (exposures1 == null)
{
exposures1 = new List<IReadOnlyDictionary<string, string>>();
}
if (exposures2 == null)
{
exposures2 = new List<IReadOnlyDictionary<string, string>>();
}

if (exposures1.Count != exposures2.Count)
{
return false;
}

var exposures2Lookup = new Dictionary<string, IReadOnlyDictionary<string, string>>();
foreach (var expo in exposures2)
{
exposures2Lookup.Add(expo["gate"], expo);
}

foreach (var expo in exposures1)
{
if (exposures2Lookup.TryGetValue(expo["gate"], out IReadOnlyDictionary<string, string> expo2))
{
if (expo["gateValue"] != expo2["gateValue"] || expo["ruleID"] != expo2["ruleID"])
{
return false;
}
}
else
{
return false;
}
}
return true;
}

private string stringifyExposures(List<IReadOnlyDictionary<string, string>> exposures)
{
var res = "[ \n";
foreach (var expo in exposures)
{
res += string.Format("Name: %s \n Value: %t \n Rule ID: %s", expo["gate"], expo["gateValue"], expo["ruleID"]);
}
res += "\n ] \n";
return res;
}
}
}
8 changes: 7 additions & 1 deletion dotnet-statsig/dotnet-statsig.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
<RootNamespace>statsig_dotnet</RootNamespace>
<PackOnBuild>true</PackOnBuild>
<PackageId>Statsig</PackageId>
<Version>1.4.1</Version>
<Version>1.4.2</Version>
<Authors>Statsig Inc.</Authors>
<PackageRequireLicenseAcceptance>true</PackageRequireLicenseAcceptance>
<Owners>Statsig Inc.,</Owners>
Expand All @@ -29,4 +29,10 @@
<PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
<PackageReference Include="IP3Country" Version="1.1.0" />
</ItemGroup>

<ItemGroup>
<AssemblyAttribute Include="System.Runtime.CompilerServices.InternalsVisibleTo">
<_Parameter1>dotnet-statsig-tests</_Parameter1>
</AssemblyAttribute>
</ItemGroup>
</Project>
9 changes: 7 additions & 2 deletions dotnet-statsig/src/Statsig/Server/Evaluation/Evaluator.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ internal ConfigEvaluation GetConfig(StatsigUser user, string configName)
private ConfigEvaluation Evaluate(StatsigUser user, ConfigSpec spec)
{
var secondaryExposures = new List<IReadOnlyDictionary<string, string>>();
var defaultRuleID = "default";
if (spec.Enabled)
{
foreach (ConfigRule rule in spec.Rules)
Expand Down Expand Up @@ -90,11 +91,15 @@ private ConfigEvaluation Evaluate(StatsigUser user, ConfigSpec spec)
}
}
}
else
{
defaultRuleID = "disabled";
}
return new ConfigEvaluation
(
EvaluationResult.Fail,
new FeatureGate(spec.Name, spec.FeatureGateDefault.Value, spec.FeatureGateDefault.RuleID, secondaryExposures),
new DynamicConfig(spec.Name, spec.DynamicConfigDefault.Value, spec.DynamicConfigDefault.RuleID, secondaryExposures)
new FeatureGate(spec.Name, spec.FeatureGateDefault.Value, defaultRuleID, secondaryExposures),
new DynamicConfig(spec.Name, spec.DynamicConfigDefault.Value, defaultRuleID, secondaryExposures)
);
}

Expand Down
12 changes: 6 additions & 6 deletions dotnet-statsig/src/Statsig/Server/ServerDriver.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ public class ServerDriver : IDisposable
bool _disposed;
RequestDispatcher _requestDispatcher;
EventLogger _eventLogger;
Evaluator _evaluator;
internal Evaluator evaluator;

public ServerDriver(string serverSecret, StatsigOptions options = null)
{
Expand All @@ -41,19 +41,19 @@ public ServerDriver(string serverSecret, StatsigOptions options = null)
Constants.SERVER_MAX_LOGGER_QUEUE_LENGTH,
Constants.SERVER_MAX_LOGGER_WAIT_TIME_IN_SEC
);
_evaluator = new Evaluator(serverSecret, options);
evaluator = new Evaluator(serverSecret, options);
}

public async Task Initialize()
{
// No op for now
await _evaluator.Initialize();
await evaluator.Initialize();
_initialized = true;
}

public void Shutdown()
{
_evaluator.Shutdown();
evaluator.Shutdown();
_eventLogger.Shutdown();
((IDisposable)this).Dispose();
}
Expand All @@ -66,7 +66,7 @@ public async Task<bool> CheckGate(StatsigUser user, string gateName)
ValidateNonEmptyArgument(gateName, "gateName");

bool result = false;
var evaluation = _evaluator.CheckGate(user, gateName);
var evaluation = evaluator.CheckGate(user, gateName);
if (evaluation?.Result == EvaluationResult.FetchFromServer)
{
var response = await _requestDispatcher.Fetch("check_gate", new Dictionary<string, object>
Expand Down Expand Up @@ -103,7 +103,7 @@ public async Task<DynamicConfig> GetConfig(StatsigUser user, string configName)
NormalizeUser(user);
ValidateNonEmptyArgument(configName, "configName");

var evaluation = _evaluator.GetConfig(user, configName);
var evaluation = evaluator.GetConfig(user, configName);
var result = evaluation?.ConfigValue;
if (evaluation == null)
{
Expand Down

0 comments on commit 239aec6

Please sign in to comment.