Release staging to r10.0

.github/workflows/bitween-api-cicd-gateway.yml

@@ -71,7 +71,11 @@ jobs:
       # (public-gateway · s9-dev-edge · cilium · letsencrypt-production-gateway).
 
       # Non-routing chart values (routing itself is generated by gateway-api mode).
-      helm-set-values: 'global.environment=Development'
+      helm-set-values: |
+        global.environment=Development
+        resources.requests.cpu=25m
+        resources.requests.memory=704Mi
+        resources.limits.memory=1408Mi
     secrets:
       # Docker image push credentials (Docker Hub)
       registry-username: ${{ secrets.DOCKERHUB_USERNAME }}

SW.Bitween.Api/Data/BitweenDbContext.cs

@@ -368,6 +368,8 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)
                 b.Property(p => p.HandlerId).HasMaxLength(200).IsUnicode(false);
                 b.Property(p => p.HandlerProperties).StoreAsJson();
                 b.Property(p => p.MapperProperties).StoreAsJson();
+                b.Property(p => p.SourceValues).StoreAsJson();
+                b.Property(p => p.SourceXchangeId).IsUnicode(false).HasMaxLength(50);
                 b.Property(p => p.InputContentType).IsUnicode(false).HasMaxLength(200);
                 b.Property(p => p.ResponseMessageTypeName).IsUnicode(false).HasMaxLength(500);
                 b.Property(p => p.FilesPrefix).HasMaxLength(200);
@@ -385,6 +387,8 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)
                 b.HasKey(p => p.Id);
                 b.Property(p => p.Id).ValueGeneratedOnAdd();
                 b.Property(p => p.References).IsSeparatorDelimited().HasMaxLength(1024);
+                b.Property(p => p.SourceValues).StoreAsJson();
+                b.Property(p => p.SourceXchangeId).IsUnicode(false).HasMaxLength(50);
                 b.HasIndex(i => i.SubscriptionId);
             });
 
@@ -447,7 +451,11 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)
 
                 b.Property(p => p.Hits).IsSeparatorDelimited().IsUnicode(false).HasMaxLength(2000);
 
-                b.HasIndex(p => p.PropertiesRaw);
+                // No b-tree on PropertiesRaw. The only query on it is the Exchanges page's substring
+                // search, a leading-wildcard LIKE no b-tree can serve (Postgres has a trigram index
+                // for it, PromotedPropertiesTrigramIndex). Having one made long values fail the
+                // save — after the delivery, so the retry delivered again — and it is what had
+                // capped the column at 450 characters on SQL Server and 255 on MySQL.
 
                 b.HasOne<Xchange>().WithOne().HasForeignKey<XchangePromotedProperties>(p => p.Id)
                     .OnDelete(DeleteBehavior.Cascade);

SW.Bitween.Api/Domain/Xchange/OnHoldXchange.cs

@@ -1,3 +1,4 @@
+using System.Collections.Generic;
 using SW.PrimitiveTypes;
 
 namespace SW.Bitween.Domain
@@ -6,7 +7,7 @@ public class OnHoldXchange:BaseEntity
     {
         private OnHoldXchange(){}
         public OnHoldXchange(Subscription subscription, string data, string fileName = null, bool badData = false, string[] references = null,
-            int? partnerId = null, string correlationId = null)
+            int? partnerId = null, string correlationId = null, XchangeSource source = null)
         {
             References = references ?? new string[] { };
             SubscriptionId = subscription.Id;
@@ -15,6 +16,8 @@ public OnHoldXchange(Subscription subscription, string data, string fileName = n
             BadData = badData;
             PartnerId = partnerId;
             CorrelationId = correlationId;
+            SourceXchangeId = source?.XchangeId;
+            SourceValues = source?.Values;
         }
         public int SubscriptionId { get; private set; }
 
@@ -32,5 +35,14 @@ public OnHoldXchange(Subscription subscription, string data, string fileName = n
         public int? PartnerId { get; private set; }
 
         public string CorrelationId { get; private set; }
+
+        /// <summary>See <see cref="Xchange.SourceXchangeId"/>.</summary>
+        public string SourceXchangeId { get; private set; }
+
+        /// <summary>The source values to run with once released. See <see cref="Xchange.SourceValues"/>.</summary>
+        public IReadOnlyDictionary<string, string> SourceValues { get; private set; }
+
+        /// <summary>What the exchange it becomes is handed, once released.</summary>
+        public XchangeSource Source => SourceXchangeId == null ? null : new(SourceXchangeId, SourceValues);
     }
 }

SW.Bitween.Api/Domain/Xchange/Xchange.cs

@@ -13,7 +13,8 @@ private Xchange()
         }
 
         public Xchange(int documentId, IWorkGroup workGroup, XchangeFile file, string[] references = null,
-            SubscriptionType subscriptionType = SubscriptionType.Internal, string correlationId = null)
+            SubscriptionType subscriptionType = SubscriptionType.Internal, string correlationId = null,
+            XchangeSource source = null)
         {
             Id = Guid.NewGuid().ToString("N");
             DocumentId = documentId;
@@ -24,6 +25,8 @@ public Xchange(int documentId, IWorkGroup workGroup, XchangeFile file, string[]
             InputContentType = file.ContentType;
             StartedOn = DateTime.UtcNow;
             CorrelationId = correlationId;
+            SourceXchangeId = source?.XchangeId;
+            SourceValues = source?.Values;
 
             XchangeCreatedEvent xchangeEvent = subscriptionType switch
             {
@@ -46,7 +49,8 @@ public Xchange(int documentId, IWorkGroup workGroup, XchangeFile file, string[]
         }
 
         public Xchange(Subscription subscription, XchangeFile file, string[] references = null,
-            string correlationId = null, Partner gatewayPartner = null, GlobalAdapterValuesSet[] globalAdapterValuesSets = null) :
+            string correlationId = null, Partner gatewayPartner = null, GlobalAdapterValuesSet[] globalAdapterValuesSets = null,
+            XchangeSource source = null) :
             this(subscription.DocumentId, subscription.WorkGroup, file, references, subscription.Type)
         {
             SubscriptionId = subscription.Id;
@@ -58,7 +62,10 @@ public Xchange(Subscription subscription, XchangeFile file, string[] references
             MapperProperties = (subscription.MapperProperties ?? new Dictionary<string, string>()).ToDictionary()
                 .Fill(gatewayPartner, globalAdapterValuesSets).WithDataSource(subscription.DataSourceId);
             HandlerProperties = (subscription.HandlerProperties ?? new Dictionary<string, string>()).ToDictionary()
-                .Fill(gatewayPartner, globalAdapterValuesSets).WithDataSource(subscription.DataSourceId);
+                .Fill(gatewayPartner, globalAdapterValuesSets).FillSource(source?.Values)
+                .WithDataSource(subscription.DataSourceId);
+            SourceXchangeId = source?.XchangeId;
+            SourceValues = source?.Values;
             CorrelationId = correlationId;
         }
 
@@ -73,6 +80,8 @@ public Xchange(Xchange xchange, XchangeFile file, IWorkGroup workGroup, bool man
             HandlerId = xchange.HandlerId;
             MapperProperties = xchange.MapperProperties;
             HandlerProperties = xchange.HandlerProperties;
+            SourceXchangeId = xchange.SourceXchangeId;
+            SourceValues = xchange.SourceValues;
             ResponseSubscriptionId = xchange.ResponseSubscriptionId;
             RetryFor = xchange.Id;
             CorrelationId = xchange.CorrelationId;
@@ -92,7 +101,10 @@ public Xchange(Subscription subscription, Xchange xchange, XchangeFile file, Par
             MapperProperties = (subscription.MapperProperties ?? new Dictionary<string, string>()).ToDictionary()
                 .Fill(gatewayPartner, globalAdapterValuesSets).WithDataSource(subscription.DataSourceId);
             HandlerProperties = (subscription.HandlerProperties ?? new Dictionary<string, string>()).ToDictionary()
-                .Fill(gatewayPartner, globalAdapterValuesSets).WithDataSource(subscription.DataSourceId);
+                .Fill(gatewayPartner, globalAdapterValuesSets).FillSource(xchange.SourceValues)
+                .WithDataSource(subscription.DataSourceId);
+            SourceXchangeId = xchange.SourceXchangeId;
+            SourceValues = xchange.SourceValues;
             ResponseSubscriptionId = subscription.ResponseSubscriptionId;
             RetryFor = xchange.Id;
             CorrelationId = xchange.CorrelationId;
@@ -114,6 +126,24 @@ public Xchange(Subscription subscription, Xchange xchange, XchangeFile file, Par
         public int? ResponseSubscriptionId { get; private set; }
         public string ResponseMessageTypeName { get; private set; }
 
+        /// <summary>
+        /// The exchange whose delivery's response started this one. Only a response subscription's
+        /// exchange has one, and a bus gateway's when its message was published as a response;
+        /// null everywhere else.
+        /// </summary>
+        public string SourceXchangeId { get; private set; }
+
+        /// <summary>
+        /// What the subscription reads from the input of <see cref="SourceXchangeId"/> — the original
+        /// document — by path, as it was when this exchange was created. A path the document had no
+        /// value at is left out.
+        /// </summary>
+        /// <remarks>
+        /// Read once and kept rather than looked up when needed, so a retry runs with the same values
+        /// as the first attempt, and they outlive the source exchange being archived.
+        /// </remarks>
+        public IReadOnlyDictionary<string, string> SourceValues { get; private set; }
+
         public string RetryFor { get; private set; }
 
         /// <summary>

SW.Bitween.Api/Domain/Xchange/XchangeSource.cs

@@ -0,0 +1,15 @@
+using System.Collections.Generic;
+
+namespace SW.Bitween.Domain
+{
+    /// <summary>
+    /// Where an exchange came from, when it came from a delivery's response: the exchange that made
+    /// the delivery, and what this one reads from that exchange's input, the original document.
+    /// </summary>
+    /// <param name="XchangeId">The exchange whose delivery's response this one runs on.</param>
+    /// <param name="Values">
+    /// Values read from the original document, by path. Null until they are read — a message off the
+    /// bus carries only the id, and each route reads what it uses.
+    /// </param>
+    public record XchangeSource(string XchangeId, IReadOnlyDictionary<string, string> Values = null);
+}

SW.Bitween.Api/Domain/XchangePromotedProperties.cs

@@ -7,25 +7,47 @@ namespace SW.Bitween.Domain
 {
     public class XchangePromotedProperties : BaseEntity<string>
     {
+        /// <summary>
+        /// The most a promoted value is stored as. A longer one is cut, ending in "…" to show it.
+        /// </summary>
+        /// <remarks>
+        /// Promoted values find and name an exchange — an order number, a customer — and they are
+        /// shown on the Exchanges page and name its archived files. A path that picks up a notes
+        /// field or a whole object would put thousands of characters in all of those. Routing is
+        /// not affected: match expressions read the payload, not these.
+        /// </remarks>
+        public const int MaxValueLength = 500;
+
         private XchangePromotedProperties()
         {
         }
 
         public XchangePromotedProperties(string xchangeId, FilterResult filterResult)
         {
             Id = xchangeId;
-            Properties = filterResult.Properties.ToDictionary();
+            Properties = filterResult.Properties.ToDictionary(p => p.Key, p => Cut(p.Value));
             Hits = filterResult.Hits?.ToArray() ?? new int[] { };
 
 
-            foreach (var (key, value) in filterResult.Properties.ToDictionary())
+            foreach (var (key, value) in Properties)
             {
                 var val = $"{key}:{value}";
                 if (string.IsNullOrEmpty(PropertiesRaw)) PropertiesRaw = val;
                 else PropertiesRaw += $",{val}";
             }
         }
 
+        /// <summary><paramref name="value"/> as it is stored: whole, or cut to <see cref="MaxValueLength"/> and "…".</summary>
+        public static string Cut(string value)
+        {
+            if (value is null || value.Length <= MaxValueLength) return value;
+
+            // Never between the two halves of a character outside the basic plane, an emoji say:
+            // half of one on its own is not valid text.
+            var length = char.IsHighSurrogate(value[MaxValueLength - 1]) ? MaxValueLength - 1 : MaxValueLength;
+            return value[..length] + "…";
+        }
+
         public IReadOnlyDictionary<string, string> Properties { get; private set; }
         public string PropertiesRaw { get; private set; }
         public int[] Hits { get; private set; }

SW.Bitween.Api/Helpers/StartupValuesFiller.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Text.RegularExpressions;
 using SW.Bitween.Domain;
 
 namespace SW.Bitween;
@@ -52,6 +53,36 @@ public static Dictionary<string, string> WithSubscription(this Dictionary<string
     }
 
 
+    /// <summary>The token prefix for a source value: <c>{{source.PATH}}</c>.</summary>
+    public const string SourceVariableNamePrefix = "source";
+
+    /// <summary>
+    /// The bus value a published response carries the id of the exchange that delivered under, so a
+    /// bus gateway route picking it up can read that exchange's input itself.
+    /// </summary>
+    public const string SourceXchangeBusValue = "bitween-source-xchange";
+
+    private static readonly Regex SourceToken = new(@"\{\{source\.([^}]+)\}\}", RegexOptions.IgnoreCase);
+
+    /// <summary>
+    /// Fills <c>{{source.PATH}}</c> from the values read out of the original document: the input of
+    /// the exchange whose delivery got the response this one runs on. Null values leave the tokens as they are — every exchange but
+    /// a response or bus gateway subscription's — and so does a path with no value, for
+    /// <see cref="SourcePathsIn"/> to find.
+    /// </summary>
+    public static Dictionary<string, string> FillSource(this Dictionary<string, string> properties,
+        IReadOnlyDictionary<string, string> sourceValues) =>
+        sourceValues == null
+            ? properties
+            : properties.Fill(new Dictionary<string, string>(sourceValues), SourceVariableNamePrefix);
+
+    /// <summary>The paths of every <c>{{source.PATH}}</c> token in <paramref name="values"/>.</summary>
+    public static IEnumerable<string> SourcePathsIn(IEnumerable<string> values) =>
+        (values ?? [])
+        .Where(v => v != null)
+        .SelectMany(v => SourceToken.Matches(v).Select(m => m.Groups[1].Value))
+        .Distinct();
+
     public static Dictionary<string, string> Fill(this IDictionary<string, string> inputTemplated,
         Partner partner, GlobalAdapterValuesSet[] globals)
     {

SW.Bitween.Api/Resources/Adapters/Catalog.cs

@@ -60,7 +60,7 @@ private async Task<IDictionary<string, StartupValue>> Describe(string adapterId)
     {
         try
         {
-            return await startupValues.Describe(adapterId);
+            return AdapterStartupValues.WithoutSecretDefaults(await startupValues.Describe(adapterId));
         }
         catch (Exception)
         {

SW.Bitween.Api/Resources/Adapters/GetProperties.cs

@@ -39,7 +39,7 @@ async public Task<object> Handle(string key)
 
             // Handle serverless adapters. The native branch above returns a different shape —
             // each key's default rather than a "key (default)" label — so it is left as it was.
-            var expected = await startupValues.Describe(decodedKey);
+            var expected = AdapterStartupValues.WithoutSecretDefaults(await startupValues.Describe(decodedKey));
 
             return expected
                 .ToList()

SW.Bitween.Api/Resources/Adapters/GetStartupValues.cs

@@ -24,7 +24,8 @@ public async Task<IDictionary<string, StartupValue>> Handle(string key)
         {
             await requestContext.EnsurePermission(dbContext, Model.Permissions.Subscriptions.View);
 
-            return await startupValues.Describe(System.Uri.UnescapeDataString(key));
+            return AdapterStartupValues.WithoutSecretDefaults(
+                await startupValues.Describe(System.Uri.UnescapeDataString(key)));
         }
     }
 }

SW.Bitween.Api/Resources/Subscriptions/GetSourcePaths.cs

@@ -0,0 +1,52 @@
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.EntityFrameworkCore;
+using SW.Bitween.Domain;
+using SW.Bitween.Model;
+using SW.PrimitiveTypes;
+
+namespace SW.Bitween.Resources.Subscriptions;
+
+/// <summary>
+/// The paths in the last document a subscription received, for the editor of a response or bus
+/// gateway subscription it feeds to offer as source values, and to warn about one that isn't there.
+/// </summary>
+/// <remarks>
+/// Needs <see cref="Model.Permissions.Exchanges.View"/>: the examples are a partner's data, read out
+/// of an exchange file. The last exchange that succeeded, because one that failed may have been
+/// refused for what it carried.
+/// </remarks>
+[HandlerName("sourcepaths")]
+public class GetSourcePaths(BitweenDbContext dbContext, RequestContext requestContext, XchangeService xchangeService)
+    : IQueryHandler<SourcePathsRequest, SourcePathsModel>
+{
+    public async Task<SourcePathsModel> Handle(SourcePathsRequest request)
+    {
+        await requestContext.EnsurePermission(dbContext, Model.Permissions.Exchanges.View);
+
+        var model = new SourcePathsModel { SubscriptionId = request.SubscriptionId };
+
+        var last = await (
+            from x in dbContext.Set<Xchange>().AsNoTracking()
+            join r in dbContext.Set<XchangeResult>() on x.Id equals r.Id
+            where x.SubscriptionId == request.SubscriptionId && r.Success
+            orderby x.StartedOn descending
+            select x
+        ).FirstOrDefaultAsync();
+        if (last == null) return model;
+
+        model.XchangeId = last.Id;
+        model.ReceivedOn = last.StartedOn;
+        try
+        {
+            model.Paths = SourceDocument.PathsIn(await xchangeService.GetFile(last, XchangeFileType.Input));
+        }
+        catch (SWValidationException)
+        {
+            // Its file is gone — deleted by retention, most likely. Nothing to offer, which is
+            // what the editor shows for a subscription that hasn't run yet.
+        }
+
+        return model;
+    }
+}