Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions docs/guide/durability/postgresql.md
Original file line number Diff line number Diff line change
Expand Up @@ -575,6 +575,17 @@ opts.MessagePartitioning.PublishToShardedPostgresqlQueues("orders", 4, topology
});
```

`PublishToShardedPostgresqlQueues()` still *listens* to the shard queues on the publishing node. If
another application consumes the queues, use `PublishOnlyToShardedPostgresqlQueues()` so this node only
provisions and publishes to them:

```cs
opts.MessagePartitioning.PublishOnlyToShardedPostgresqlQueues("orders", 4, topology =>
{
topology.MessagesImplementing<IOrderMessage>();
});
```

::: info Long base names
PostgreSQL truncates identifiers at `NAMEDATALEN` (63 characters by default). Wolverine runs every
queue table name through Weasel's deterministic shortening helper, so a long sharded base name still
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -214,3 +214,79 @@ public static class PgLetterHandler
public static void Handle(PgLogC message, Envelope envelope) => Received.Add((message.Id, envelope.Destination));
public static void Handle(PgLogD message, Envelope envelope) => Received.Add((message.Id, envelope.Destination));
}

/// <summary>
/// GH-5088: the publishing application must not listen to the shard queues, so a separate
/// application can consume them later.
/// </summary>
public class publish_only_to_sharded_postgresql_queues : IAsyncLifetime
{
private IHost _host = null!;

public async ValueTask InitializeAsync()
{
_host = await Host.CreateDefaultBuilder()
.UseWolverine(opts =>
{
opts.Durability.Mode = DurabilityMode.Solo;

opts.UsePostgresqlPersistenceAndTransport(Servers.PostgresConnectionString, "pletters_pub_only",
transportSchema: "pletters_pub_only_queues")
.AutoProvision()
.AutoPurgeOnStartup();

// Deliberately no handler for the messages: this node only publishes
opts.Discovery.DisableConventionalDiscovery();

opts.MessagePartitioning.ByMessage<IPgLetterMessage>(x => x.Id.ToString());

opts.MessagePartitioning.PublishOnlyToShardedPostgresqlQueues("plettersonly", 4, topology =>
{
topology.MessagesImplementing<IPgLetterMessage>();
topology.MaxDegreeOfParallelism = PartitionSlots.Five;
});
}).StartAsync();
}

public async ValueTask DisposeAsync()
{
await _host.StopAsync();
_host.Dispose();
}

[Fact]
public async Task the_shard_queues_are_not_listened_to()
{
var runtime = _host.GetRuntime();
var queues = runtime.Options.PostgresqlTransport().Queues
.Where(x => x.EndpointName.StartsWith("plettersonly")).ToArray();

queues.Length.ShouldBe(4);
queues.ShouldAllBe(x => !x.IsListener);
}

[Fact]
public async Task published_messages_stay_in_the_queue_tables_unprocessed()
{
var bus = _host.MessageBus();
for (var i = 0; i < 10; i++)
{
await bus.PublishAsync(new PgLogA(Guid.NewGuid()));
}

await Task.Delay(2.Seconds(), TestContext.Current.CancellationToken);

var transport = _host.GetRuntime().Options.PostgresqlTransport();
var total = 0L;
await using var conn = new Npgsql.NpgsqlConnection(Servers.PostgresConnectionString);
await conn.OpenAsync(TestContext.Current.CancellationToken);
foreach (var queue in transport.Queues.Where(x => x.EndpointName.StartsWith("plettersonly")))
{
await using var cmd = conn.CreateCommand();
cmd.CommandText = $"select count(*) from pletters_pub_only_queues.{queue.QueueTable.Identifier.Name}";
total += (long)(await cmd.ExecuteScalarAsync(TestContext.Current.CancellationToken))!;
}

total.ShouldBe(10);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -315,6 +315,27 @@ public static MessagePartitioningRules PublishToShardedPostgresqlQueues(this Mes
return rules;
}

/// <summary>
/// Shard message publishing across a set of PostgreSQL queues named baseName1, baseName2, and so on,
/// but never listen to them on this node. Use this when another application consumes the queues.
/// <see cref="PartitionedMessageTopology{TListener,TSubscriber}.ConfigureListening" /> and
/// MaxDegreeOfParallelism have no effect on a publish-only topology.
/// </summary>
public static MessagePartitioningRules PublishOnlyToShardedPostgresqlQueues(this MessagePartitioningRules rules,
string baseName, int numberOfEndpoints, Action<PartitionedMessageTopologyWithDatabaseQueues> configure)
{
rules.AddPublishingTopology((opts, _) =>
{
var topology = new PartitionedMessageTopologyWithDatabaseQueues(opts, baseName, numberOfEndpoints, true);
configure(topology);
topology.AssertValidity();

return topology;
});

return rules;
}

/// <summary>
/// Use sharded PostgreSQL queues for global partitioned message processing.
/// Queues will be named baseName1, baseName2, etc.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,18 @@ public PartitionedMessageTopologyWithDatabaseQueues(WolverineOptions options, Pa
MaxDegreeOfParallelism = PartitionSlots.Five;
}

/// <summary>
/// Build a publish-only topology: the shard queues are provisioned and published to, but this
/// node never listens to them, so another application can consume them.
/// </summary>
public PartitionedMessageTopologyWithDatabaseQueues(WolverineOptions options, string baseName,
int numberOfEndpoints, bool publishOnly) : base(options, publishOnly ? null : PartitionSlots.Five, baseName,
numberOfEndpoints)
{
PublishOnly = publishOnly;
if (!publishOnly) MaxDegreeOfParallelism = PartitionSlots.Five;
}

protected override Endpoint buildEndpoint(WolverineOptions options, string name)
{
var transport = options.PostgresqlTransport();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,13 @@ public abstract class PartitionedMessageTopology<TListener, TSubscriber> : Parti
{
private PartitionSlots _listeningSlots;

/// <summary>
/// When true, this topology only publishes to its sharded endpoints and never registers a listener
/// for them on this node. <see cref="ConfigureListening" /> and <see cref="MaxDegreeOfParallelism" />
/// then have no listening effect. Set it in a derived constructor.
/// </summary>
protected bool PublishOnly { get; set; }

public PartitionedMessageTopology(WolverineOptions options, PartitionSlots? listeningSlots, string baseName, int numberOfEndpoints) : base(options, listeningSlots, baseName, numberOfEndpoints)
{
if (listeningSlots.HasValue)
Expand Down Expand Up @@ -60,6 +67,8 @@ public void ConfigureSender(Action<TSubscriber> configure)

public void ConfigureListening(Action<TListener> configure)
{
if (PublishOnly) return;

foreach (var name in _names)
{
var listener = buildListener(_options, name);
Expand Down