Comments (12)
I debugged the orleans source code and found that line 169 if (membershipSnapshot.Entries.TryGetValue(_localSiloDetails.SiloAddress, out var membershipEntry))
of class LocalSiloHealthMonitor
always returns false
from orleans.redis.
demo:
public class Program
{
static async Task<int> Main(string[] args)
{
// 启动服务
try
{
var host = await StartSilo();
Console.WriteLine("Silo is ready!");
}
catch (Exception ex)
{
Console.WriteLine(ex);
Console.ReadLine();
return 1;
}
// 启动客户端
try
{
var client = await StartClientWithRetries();
await DoClientWork(client);
}
catch (Exception e)
{
Console.WriteLine(e);
return 1;
}
Console.ReadKey();
return 0;
}
private static async Task<ISiloHost> StartSilo()
{
var builder = new SiloHostBuilder()
.Configure<ClusterOptions>(options =>
{
options.ClusterId = "testcluster";
options.ServiceId = "testcluster";
})
.ConfigureEndpoints(IPAddress.Loopback, 22222, 33333,listenOnAnyHostAddress: true)
.UseRedisClustering(opt =>
{
opt.ConnectionString = "192.168.103.113:6379,password=Qbkj@2020";
opt.Database = 13;
})
.AddMemoryGrainStorageAsDefault()
.ConfigureApplicationParts(parts => parts.AddApplicationPart(typeof(HelloGrain).Assembly).WithReferences())
.ConfigureLogging(logging => logging.AddConsole());
var host = builder.Build();
await host.StartAsync();
return host;
}
private static async Task<IClusterClient> StartClientWithRetries(int initializeAttemptsBeforeFailing = 5)
{
int attempt = 0;
IClusterClient client;
while (true)
{
try
{
client = new ClientBuilder()
.Configure<ClusterOptions>(options =>
{
options.ClusterId = "testcluster";
options.ServiceId = "testcluster";
})
.UseRedisClustering(opt =>
{
opt.ConnectionString = "192.168.103.113:6379,password=Qbkj@2020";
opt.Database = 13;
})
.ConfigureApplicationParts(parts => parts.AddApplicationPart(typeof(IHello).Assembly).WithReferences())
.ConfigureLogging(logging => logging.AddConsole())
.Build();
await client.Connect();
Console.WriteLine("Client successfully connect to silo host");
break;
}
catch (SiloUnavailableException)
{
attempt++;
Console.WriteLine($"Attempt {attempt} of {initializeAttemptsBeforeFailing} failed to initialize the Orleans client.");
if (attempt > initializeAttemptsBeforeFailing)
{
throw;
}
await Task.Delay(TimeSpan.FromSeconds(4));
}
}
return client;
}
private static async Task DoClientWork(IClusterClient client)
{
var friend = client.GetGrain<IHello>(0);
for (int i = 0; i < 10; i++)
{
var response = await friend.SayHello("Good morning, my friend!");
Console.WriteLine("\n\n{0}\n\n", response);
}
}
}
from orleans.redis.
I Solved the problem, but I don't know why
from orleans.redis.
No Solved, still have the problem
from orleans.redis.
I tested ado cluster and there is no such problem
from orleans.redis.
@qzwzqty are you able to reproduce this in a test? If so, that will make it easier for us to fix it
from orleans.redis.
demo:
public class Program { static async Task<int> Main(string[] args) { // 启动服务 try { var host = await StartSilo(); Console.WriteLine("Silo is ready!"); } catch (Exception ex) { Console.WriteLine(ex); Console.ReadLine(); return 1; } // 启动客户端 try { var client = await StartClientWithRetries(); await DoClientWork(client); } catch (Exception e) { Console.WriteLine(e); return 1; } Console.ReadKey(); return 0; } private static async Task<ISiloHost> StartSilo() { var builder = new SiloHostBuilder() .Configure<ClusterOptions>(options => { options.ClusterId = "testcluster"; options.ServiceId = "testcluster"; }) .ConfigureEndpoints(22222, 33333) .UseRedisClustering(opt => { opt.ConnectionString = "192.168.103.113:6379,password=Qbkj@2020"; opt.Database = 13; }) .AddMemoryGrainStorageAsDefault() .ConfigureApplicationParts(parts => parts.AddApplicationPart(typeof(HelloGrain).Assembly).WithReferences()) .ConfigureLogging(logging => logging.AddConsole()); var host = builder.Build(); await host.StartAsync(); return host; } private static async Task<IClusterClient> StartClientWithRetries(int initializeAttemptsBeforeFailing = 5) { int attempt = 0; IClusterClient client; while (true) { try { client = new ClientBuilder() .Configure<ClusterOptions>(options => { options.ClusterId = "testcluster"; options.ServiceId = "testcluster"; }) .UseRedisClustering(opt => { opt.ConnectionString = "192.168.103.113:6379,password=Qbkj@2020"; opt.Database = 13; }) .ConfigureApplicationParts(parts => parts.AddApplicationPart(typeof(IHello).Assembly).WithReferences()) .ConfigureLogging(logging => logging.AddConsole()) .Build(); await client.Connect(); Console.WriteLine("Client successfully connect to silo host"); break; } catch (SiloUnavailableException) { attempt++; Console.WriteLine($"Attempt {attempt} of {initializeAttemptsBeforeFailing} failed to initialize the Orleans client."); if (attempt > initializeAttemptsBeforeFailing) { throw; } await Task.Delay(TimeSpan.FromSeconds(4)); } } return client; } private static async Task DoClientWork(IClusterClient client) { var friend = client.GetGrain<IHello>(0); for (int i = 0; i < 10; i++) { var response = await friend.SayHello("Good morning, my friend!"); Console.WriteLine("\n\n{0}\n\n", response); } } }
@ReubenBond This demo can reproduce. This is the client demo and silo demo of this repository
from orleans.redis.
from orleans.redis.
After a while
[2021-10-20 11:22:46 ERR Orleans.Runtime.MembershipService.MembershipAgent] Failed to update table entry for this silo, will retry shortly: Orleans.Clustering.Redis.RedisClusteringException: Could not find a value for the key S10.42.9.231:30000:372395638
at Orleans.Clustering.Redis.RedisMembershipTable.UpdateIAmAlive(MembershipEntry entry)
at Orleans.Runtime.MembershipService.MembershipTableManager.UpdateIAmAlive()
at Orleans.Runtime.MembershipService.MembershipAgent.UpdateIAmAlive()
from orleans.redis.
Starting to get same problem which is weird, I didnt update anything and it was stable for months.
Both redis and orleans are in kubernetes. According to the healtcheck the redis was alive.
Orleans version is 3.5.0
[2021-12-15 07:49:29 ERR (Orleans.Runtime.MembershipService.MembershipAgent)] Failed to update table entry for this silo, will retry shortly: Orleans.Clustering.Redis.RedisClusteringException: Could not find a value for the key S10.106.2.150:31017:377174548
at Orleans.Clustering.Redis.RedisMembershipTable.UpdateIAmAlive(MembershipEntry entry)
at Orleans.Runtime.MembershipService.MembershipTableManager.UpdateIAmAlive()
at Orleans.Runtime.MembershipService.MembershipAgent.UpdateIAmAlive()
�[41m�[30mfail�[39m�[22m�[49m: Orleans.Runtime.MembershipService.MembershipAgent[0]
Failed to update table entry for this silo, will retry shortly: "Orleans.Clustering.Redis.RedisClusteringException: Could not find a value for the key S10.106.2.150:31017:377174548
at Orleans.Clustering.Redis.RedisMembershipTable.UpdateIAmAlive(MembershipEntry entry)
at Orleans.Runtime.MembershipService.MembershipTableManager.UpdateIAmAlive()
at Orleans.Runtime.MembershipService.MembershipAgent.UpdateIAmAlive()"
[2021-12-15 07:49:29 ERR (Orleans.Runtime.MembershipService.MembershipAgent)] Failed to update table entry for this silo, will retry shortly: Orleans.Clustering.Redis.RedisClusteringException: Could not find a value for the key S10.106.2.150:31017:377174548
at Orleans.Clustering.Redis.RedisMembershipTable.UpdateIAmAlive(MembershipEntry entry)
at Orleans.Runtime.MembershipService.MembershipTableManager.UpdateIAmAlive()
at Orleans.Runtime.MembershipService.MembershipAgent.UpdateIAmAlive()
�[41m�[30mfail�[39m�[22m�[49m: Orleans.Runtime.MembershipService.MembershipAgent[0]
Failed to update table entry for this silo, will retry shortly: "Orleans.Clustering.Redis.RedisClusteringException: Could not find a value for the key S10.106.2.150:31017:377174548
at Orleans.Clustering.Redis.RedisMembershipTable.UpdateIAmAlive(MembershipEntry entry)
at Orleans.Runtime.MembershipService.MembershipTableManager.UpdateIAmAlive()
at Orleans.Runtime.MembershipService.MembershipAgent.UpdateIAmAlive()"
�[41m�[30mfail�[39m�[22m�[49m: Orleans.Runtime.MembershipService.MembershipAgent[0]
Failed to update table entry for this silo, will retry shortly: "Orleans.Clustering.Redis.RedisClusteringException: Could not find a value for the key S10.106.2.150:31017:377174548
at Orleans.Clustering.Redis.RedisMembershipTable.UpdateIAmAlive(MembershipEntry entry)
at Orleans.Runtime.MembershipService.MembershipTableManager.UpdateIAmAlive()
at Orleans.Runtime.MembershipService.MembershipAgent.UpdateIAmAlive()"
[2021-12-15 07:49:29 ERR (Orleans.Runtime.MembershipService.MembershipAgent)] Failed to update table entry for this silo, will retry shortly: Orleans.Clustering.Redis.RedisClusteringException: Could not find a value for the key S10.106.2.150:31017:377174548
at Orleans.Clustering.Redis.RedisMembershipTable.UpdateIAmAlive(MembershipEntry entry)
at Orleans.Runtime.MembershipService.MembershipTableManager.UpdateIAmAlive()
at Orleans.Runtime.MembershipService.MembershipAgent.UpdateIAmAlive()
[2021-12-15 07:49:29 ERR (Orleans.Runtime.MembershipService.MembershipAgent)] Failed to update table entry for this silo, will retry shortly: Orleans.Clustering.Redis.RedisClusteringException: Could not find a value for the key S10.106.2.150:31017:377174548
at Orleans.Clustering.Redis.RedisMembershipTable.UpdateIAmAlive(MembershipEntry entry)
at Orleans.Runtime.MembershipService.MembershipTableManager.UpdateIAmAlive()
at Orleans.Runtime.MembershipService.MembershipAgent.UpdateIAmAlive()
�[41m�[30mfail�[39m�[22m�[49m: Orleans.Runtime.MembershipService.MembershipAgent[0]
Failed to update table entry for this silo, will retry shortly: "Orleans.Clustering.Redis.RedisClusteringException: Could not find a value for the key S10.106.2.150:31017:377174548
at Orleans.Clustering.Redis.RedisMembershipTable.UpdateIAmAlive(MembershipEntry entry)
at Orleans.Runtime.MembershipService.MembershipTableManager.UpdateIAmAlive()
at Orleans.Runtime.MembershipService.MembershipAgent.UpdateIAmAlive()"
[2021-12-15 07:49:29 ERR (Orleans.Runtime.MembershipService.MembershipAgent)] Failed to update table entry for this silo, will retry shortly: Orleans.Clustering.Redis.RedisClusteringException: Could not find a value for the key S10.106.2.150:31017:377174548
at Orleans.Clustering.Redis.RedisMembershipTable.UpdateIAmAlive(MembershipEntry entry)
at Orleans.Runtime.MembershipService.MembershipTableManager.UpdateIAmAlive()
at Orleans.Runtime.MembershipService.MembershipAgent.UpdateIAmAlive()
�[41m�[30mfail�[39m�[22m�[49m: Orleans.Runtime.MembershipService.MembershipAgent[0]
Failed to update table entry for this silo, will retry shortly: "Orleans.Clustering.Redis.RedisClusteringException: Could not find a value for the key S10.106.2.150:31017:377174548
at Orleans.Clustering.Redis.RedisMembershipTable.UpdateIAmAlive(MembershipEntry entry)
at Orleans.Runtime.MembershipService.MembershipTableManager.UpdateIAmAlive()
at Orleans.Runtime.MembershipService.MembershipAgent.UpdateIAmAlive()"
[2021-12-15 07:49:30 ERR (Orleans.Runtime.MembershipService.MembershipAgent)] Failed to update table entry for this silo, will retry shortly: Orleans.Clustering.Redis.RedisClusteringException: Could not find a value for the key S10.106.2.150:31017:377174548
at Orleans.Clustering.Redis.RedisMembershipTable.UpdateIAmAlive(MembershipEntry entry)
at Orleans.Runtime.MembershipService.MembershipTableManager.UpdateIAmAlive()
at Orleans.Runtime.MembershipService.MembershipAgent.UpdateIAmAlive()
�[41m�[30mfail�[39m�[22m�[49m: Orleans.Runtime.MembershipService.MembershipAgent[0]
Failed to update table entry for this silo, will retry shortly: "Orleans.Clustering.Redis.RedisClusteringException: Could not find a value for the key S10.106.2.150:31017:377174548
at Orleans.Clustering.Redis.RedisMembershipTable.UpdateIAmAlive(MembershipEntry entry)
at Orleans.Runtime.MembershipService.MembershipTableManager.UpdateIAmAlive()
at Orleans.Runtime.MembershipService.MembershipAgent.UpdateIAmAlive()"
[2021-12-15 07:49:30 ERR (Orleans.Runtime.MembershipService.MembershipAgent)] Failed to update table entry for this silo, will retry shortly: Orleans.Clustering.Redis.RedisClusteringException: Could not find a value for the key S10.106.2.150:31017:377174548
at Orleans.Clustering.Redis.RedisMembershipTable.UpdateIAmAlive(MembershipEntry entry)
at Orleans.Runtime.MembershipService.MembershipTableManager.UpdateIAmAlive()
at Orleans.Runtime.MembershipService.MembershipAgent.UpdateIAmAlive()
�[41m�[30mfail�[39m�[22m�[49m: Orleans.Runtime.MembershipService.MembershipAgent[0]
Failed to update table entry for this silo, will retry shortly: "Orleans.Clustering.Redis.RedisClusteringException: Could not find a value for the key S10.106.2.150:31017:377174548
at Orleans.Clustering.Redis.RedisMembershipTable.UpdateIAmAlive(MembershipEntry entry)
at Orleans.Runtime.MembershipService.MembershipTableManager.UpdateIAmAlive()
at Orleans.Runtime.MembershipService.MembershipAgent.UpdateIAmAlive()"
[2021-12-15 07:49:30 ERR (Orleans.Runtime.MembershipService.MembershipAgent)] Failed to update table entry for this silo, will retry shortly: Orleans.Clustering.Redis.RedisClusteringException: Could not find a value for the key S10.106.2.150:31017:377174548
at Orleans.Clustering.Redis.RedisMembershipTable.UpdateIAmAlive(MembershipEntry entry)
at Orleans.Runtime.MembershipService.MembershipTableManager.UpdateIAmAlive()
at Orleans.Runtime.MembershipService.MembershipAgent.UpdateIAmAlive()
�[41m�[30mfail�[39m�[22m�[49m: Orleans.Runtime.MembershipService.MembershipAgent[0]
Failed to update table entry for this silo, will retry shortly: "Orleans.Clustering.Redis.RedisClusteringException: Could not find a value for the key S10.106.2.150:31017:377174548
at Orleans.Clustering.Redis.RedisMembershipTable.UpdateIAmAlive(MembershipEntry entry)
at Orleans.Runtime.MembershipService.MembershipTableManager.UpdateIAmAlive()
at Orleans.Runtime.MembershipService.MembershipAgent.UpdateIAmAlive()"
[2021-12-15 07:49:30 ERR (Orleans.Runtime.MembershipService.MembershipAgent)] Failed to update table entry for this silo, will retry shortly: Orleans.Clustering.Redis.RedisClusteringException: Could not find a value for the key S10.106.2.150:31017:377174548
at Orleans.Clustering.Redis.RedisMembershipTable.UpdateIAmAlive(MembershipEntry entry)
at Orleans.Runtime.MembershipService.MembershipTableManager.UpdateIAmAlive()
at Orleans.Runtime.MembershipService.MembershipAgent.UpdateIAmAlive()
�[41m�[30mfail�[39m�[22m�[49m: Orleans.Runtime.MembershipService.MembershipAgent[0]
Failed to update table entry for this silo, will retry shortly: "Orleans.Clustering.Redis.RedisClusteringException: Could not find a value for the key S10.106.2.150:31017:377174548
at Orleans.Clustering.Redis.RedisMembershipTable.UpdateIAmAlive(MembershipEntry entry)
at Orleans.Runtime.MembershipService.MembershipTableManager.UpdateIAmAlive()
at Orleans.Runtime.MembershipService.MembershipAgent.UpdateIAmAlive()"
[2021-12-15 07:49:31 ERR (Orleans.Runtime.MembershipService.MembershipAgent)] Failed to update table entry for this silo, will retry shortly: Orleans.Clustering.Redis.RedisClusteringException: Could not find a value for the key S10.106.2.150:31017:377174548
at Orleans.Clustering.Redis.RedisMembershipTable.UpdateIAmAlive(MembershipEntry entry)
at Orleans.Runtime.MembershipService.MembershipTableManager.UpdateIAmAlive()
at Orleans.Runtime.MembershipService.MembershipAgent.UpdateIAmAlive()
�[41m�[30mfail�[39m�[22m�[49m: Orleans.Runtime.MembershipService.MembershipAgent[0]
Failed to update table entry for this silo, will retry shortly: "Orleans.Clustering.Redis.RedisClusteringException: Could not find a value for the key S10.106.2.150:31017:377174548
at Orleans.Clustering.Redis.RedisMembershipTable.UpdateIAmAlive(MembershipEntry entry)
at Orleans.Runtime.MembershipService.MembershipTableManager.UpdateIAmAlive()
at Orleans.Runtime.MembershipService.MembershipAgent.UpdateIAmAlive()"
from orleans.redis.
As I understand it could not find the record in redis
which is weird, sadly the silo and redis was restarted before I could look if the record exist or not in redis.
Maybe it would make sense to make it this way, if the record wasnt found then instead to try to update it make an new record?
from orleans.redis.
Side note: was able to reproduce it on dev env by simply removing the redis record. But I can't be sure if this was my case, because silo was restarted by coworker before I had a chance to look in redis
from orleans.redis.
Related Issues (17)
- JSON deserialization of grain states does not bind GrainReferences to runtime HOT 4
- Redis Streaming HOT 1
- Does not work with Co-Hosting HOT 2
- Redis Clustering in Docker Swarm, Silo's wont find each other HOT 2
- All keys deleted after Redis server rebooted unexpectedly HOT 10
- RedisGrainStorage.ReadStateAsync() should use Array.Length instead of Linq.Enumerable.Count()
- GrainReference not deleted from redis HOT 1
- FLUSHALL required between running different code HOT 1
- [Question] Can it be used on ISiloBuilder? HOT 1
- [Question] intention of redis key format (ServiceId/ClusterId)
- Regular null reference exceptions HOT 1
- Grain multiple states HOT 1
- Orleans 4.0 compatibility
- Unable to resolve service for type Orleans.Runtime.IPersistentState HOT 3
- 'ISiloBuilder' does not contain a definition for 'AddRedisGrainStorage' HOT 5
- Cannot link redis with password HOT 1
Recommend Projects
-
React
A declarative, efficient, and flexible JavaScript library for building user interfaces.
-
Vue.js
🖖 Vue.js is a progressive, incrementally-adoptable JavaScript framework for building UI on the web.
-
Typescript
TypeScript is a superset of JavaScript that compiles to clean JavaScript output.
-
TensorFlow
An Open Source Machine Learning Framework for Everyone
-
Django
The Web framework for perfectionists with deadlines.
-
Laravel
A PHP framework for web artisans
-
D3
Bring data to life with SVG, Canvas and HTML. 📊📈🎉
-
Recommend Topics
-
javascript
JavaScript (JS) is a lightweight interpreted programming language with first-class functions.
-
web
Some thing interesting about web. New door for the world.
-
server
A server is a program made to process requests and deliver data to clients.
-
Machine learning
Machine learning is a way of modeling and interpreting data that allows a piece of software to respond intelligently.
-
Visualization
Some thing interesting about visualization, use data art
-
Game
Some thing interesting about game, make everyone happy.
Recommend Org
-
Facebook
We are working to build community through open source technology. NB: members must have two-factor auth.
-
Microsoft
Open source projects and samples from Microsoft.
-
Google
Google ❤️ Open Source for everyone.
-
Alibaba
Alibaba Open Source for everyone
-
D3
Data-Driven Documents codes.
-
Tencent
China tencent open source team.
from orleans.redis.