configure invoice formats and output framing

DataAgent/CAPABILITIES.md

@@ -30,8 +30,8 @@ owned options are the keys the adapter interprets itself. other arguments belong
 | `fmt/csv` | `Path`, `Encoding`, `StripQuotes` | `ConvertTo-Csv` |
 | `fmt/xlsx` | `Path` | `Export-Excel` |
 | `fmt/custom` | `Module`, `Path` | `ConvertTo-Custom` |
-| `fmt/pdi` | `Path`, `Values`, `Key`, `Head`, `Type`, `H`, `D`, `I` | none |
-| `fmt/x12-210` | `Path`, `Values`, `Key`, `Detail`, `Head`, `Invoice`, `Line`, `Close`, `Se`, `SeExtra`, `Tail` | none |
+| `fmt/pdi` | `Path`, `Values`, `Key`, `Head`, `Type`, `H`, `D`, `I`, `Footer` | none |
+| `fmt/x12-210` | `Path`, `Values`, `Key`, `Detail`, `Head`, `Invoice`, `Line`, `Close`, `Se`, `SeExtra`, `Tail`, `Control`, `Dates`, `Separator`, `Terminator`, `SeDetailCount` | none |
 | `fmt/dtn` | `Path`, `Values`, `Head`, `Start`, `Open`, `Header`, `Detail`, `Line`, `Close`, `Tail` | none |
 | `dst/email` | none | `Send-FileViaEmail` |
 | `dst/sftp` | `connect`, `send`, `attempts`, `delay` | `New-SFTPSession`, `Set-SFTPItem` |
@@ -51,6 +51,20 @@ owned options are the keys the adapter interprets itself. other arguments belong
 
 ## custom adapters
 
+### text frames and invoice variants
+
+`fmt.Prepend` and `fmt.Append` are line templates outside `fmt.args`, so they do not leak into helper splats. the runner adds them after formatting and before any destination. this is text-only, using args.Encoding or utf8NoBOM; XLSX and ZIP paths refuse. custom binary outputs must not use framing. no framing keys means no extra file read/write.
+
+frames support the same substitutions as invoices, with fields from the first source row, `now` and `sum`. `fmt.Sum` maps a name to `{ Property: 'amount', UniqueBy: 'invoice_id' }`. `UniqueBy` is optional: supplied means take the first row of each group before summing, omitted means sum all rows. sums cast values to decimal like the extracted report. keep repeated invoice totals out of an all-row sum. for example Prepend `['run {now:f}', 'total {sum.total:0.00}']` and Sum `{ total: { Property: 'amount', UniqueBy: 'invoice_id' } }`. invoice args.Values.now can supply the filing clock for the invoice and its frame; JSON ISO date strings are accepted. other formats use the current clock, or a dated source field in a template.
+
+pdi `Footer` is a line after each invoice, using that invoice's header fields. an empty H omits the header. an empty Type sends all detail rows through D. together these let a configured line-per-charge layout have one invoice total without a new formatter. defaults still select d/i and write H/D/I.
+
+x12 template entries can stay strings or use `{ Text: ['N9*{ref}', 'L0*{qty}'], When: { Field: 'kind', Is: 'A', Not: true } }`. Is compares strings case-insensitively. without Is, a nonempty value matches. Not negates the test. conditions read the row; skipped segments do not count toward SE. Separator replaces literal `*` in templates before substitutions, never stars in data values. Terminator is appended to each emitted X12 segment. use outer text frames for non-X12 front matter.
+
+x12 Control is a template: omit it for the existing hour-of-year scheme, use `1{now:yyMMdd}00` for a date scheme or `{now:yy}{day:000}{now:HHmm}` for day-of-year and time. Values.control still overrides the result. Dates maps names to day offsets, for example `{ due: 7 }` then `{dates.due:yyyyMMdd}`. the filing clock drives all derived values. existing Se and Tail templates already handle padded counts and fixed GE fields. SeExtra adds a legacy count offset. optional SeDetailCount replaces the counted detail-segment total with that fixed number per detail row; it is an explicit legacy receiver quirk, not the default or a standards claim.
+
+invoice groups keep their first-seen order, and each invoice's details keep their input order. the source owns any sorting. pdi no longer sorts invoices by key or moves D records before I records. grouping still puts an invoice's rows together if the source interleaves invoice keys.
+
 invoice substitutions without a format preserve PowerShell's string interpolation, including dates and numbers. alignment pads that string. an explicit `:format` instead formats the original typed value using .NET's current-culture formatting.
 
 set `adapter` to a `.ps1` path, relative to the job directory or absolute. no registration or core edit. each script receives `-Data` and `-Options`:

DataAgent/DataAgent.psd1

@@ -1,6 +1,6 @@
 @{
     RootModule = 'DataAgent.psm1'
-    ModuleVersion = '0.9.0'
+    ModuleVersion = '0.10.0'
     GUID = '32662c0b-0a5d-49ac-8c62-8f7ae05f79c0'
     Author = 'Roy Ashbrook'
     Copyright = '(c) 2026 Roy Ashbrook. MIT License.'
@@ -10,13 +10,13 @@
     CmdletsToExport = @()
     VariablesToExport = @()
     AliasesToExport = @()
-    FileList = @('DataAgent.psd1', 'DataAgent.psm1', 'LICENSE', 'CAPABILITIES.md', 'src/sql.ps1', 'src/csv.ps1', 'src/cosmos.ps1', 'fmt/csv.ps1', 'fmt/xlsx.ps1', 'fmt/custom.ps1', 'fmt/pdi.ps1', 'fmt/x12-210.ps1', 'fmt/dtn.ps1', 'fmt/invoice/template.ps1', 'fmt/invoice/pdi.psd1', 'fmt/invoice/x12-210.psd1', 'fmt/invoice/dtn.psd1', 'dst/email.ps1', 'dst/sftp.ps1', 'dst/ftp.ps1', 'dst/ftps.ps1')
+    FileList = @('DataAgent.psd1', 'DataAgent.psm1', 'LICENSE', 'CAPABILITIES.md', 'src/sql.ps1', 'src/csv.ps1', 'src/cosmos.ps1', 'fmt/csv.ps1', 'fmt/xlsx.ps1', 'fmt/custom.ps1', 'fmt/pdi.ps1', 'fmt/x12-210.ps1', 'fmt/dtn.ps1', 'fmt/invoice/template.ps1', 'fmt/invoice/pdi.psd1', 'fmt/invoice/x12-210.psd1', 'fmt/invoice/dtn.psd1', 'fmt/text/frame.ps1', 'dst/email.ps1', 'dst/sftp.ps1', 'dst/ftp.ps1', 'dst/ftps.ps1')
     PrivateData = @{
         PSData = @{
             Tags = @('ETL', 'Data', 'CSV', 'Automation', 'PSEdition_Core', 'Windows', 'Linux', 'MacOS')
             LicenseUri = 'https://github.com/royashbrook/DataAgent/blob/main/LICENSE'
             ProjectUri = 'https://github.com/royashbrook/DataAgent'
-            ReleaseNotes = 'Add fmt/pdi, fmt/x12-210 and fmt/dtn invoice layout baselines with configurable templates and values. Plain substitutions preserve PowerShell interpolation; explicit formats retain typed .NET formatting. Runner, source, destination and dependency behavior is unchanged.'
+            ReleaseNotes = 'Preserve invoice input order. Add configured x12 conditions, control/date schemes, separators, terminators and legacy counts. Add text prepend/append with optional distinct-key sums, and per-invoice footer templates. No helper pin or destination changes. Re-prove receiver output before adoption.'
         }
     }
 }

DataAgent/DataAgent.psm1

@@ -68,6 +68,10 @@ function Invoke-DataAgent {
             if ($Config.file_format) { $options.Path = $Config.file_format -f (Get-Date) }
             $file = $options.Path
             & ($adapters.fmt[0]) -Data $data -Options $options
+            if ($null -ne $Config.fmt.Prepend -or $null -ne $Config.fmt.Append) {
+                . "$PSScriptRoot/fmt/text/frame.ps1"
+                Add-FormatFrame $Config.fmt $data $options
+            }
             l 'Use Data'
             for ($i = 0; $i -lt $adapters.dst.Count; $i++) { & ($adapters.dst[$i]) -Data $file -Options @($Config.dst)[$i].args }
             l 'End'

DataAgent/fmt/invoice/pdi.psd1

@@ -3,6 +3,7 @@
     Head = 'ivh*'
     Type = 'dori'
     Values = @{}
+    Footer = ''
     H = '"H","{sender_id}","{ivh_invoicenumber}","{ivh_billdate}",,,,,,"{ivh_totalcharge}","N"'
     D = '"D","{ivd_refnum}","{ivh_consignee}","{cmd_code}","{not_text}","{sender_id}",,"{ivd_quantity}","{ivd_wgt}","{ivd_rate}",,"{ivd_charge}",'
     I = '"I","{cht_itemcode}","{ivd_charge}",'

DataAgent/fmt/invoice/template.ps1

@@ -3,6 +3,8 @@
 function Get-InvoiceOptions($Overrides, $Name) {
     $options = Import-PowerShellDataFile "$PSScriptRoot/$Name.psd1"
     foreach ($key in $Overrides.Keys) { $options[$key] = $Overrides[$key] }
+    $options.Values = @{} + $options.Values
+    $options.Values.now = if ($options.Values.ContainsKey('now')) { [datetime]$options.Values.now } else { Get-Date }
     $options
 }
 

DataAgent/fmt/invoice/x12-210.psd1

@@ -2,6 +2,11 @@
     Key = 'b302'
     Detail = @('l5*', 'l0*', 'l1*')
     Values = @{}
+    Control = ''
+    Dates = @{}
+    Separator = '*'
+    Terminator = ''
+    SeDetailCount = $null
     Head = @(
         'ISA*00*          *00*          *02*{sender_id,-15}*ZZ*{receiver_id,-15}*{now:yyMMdd}*{now:HHmm}*U*00401*{control:000000000}*0*P*>'
         'GS*IM*{sender_id}*{receiver_id}*{now:yyyyMMdd}*{now:HHmm}*{control}*X*004010'

DataAgent/fmt/pdi.ps1

@@ -3,12 +3,12 @@ param($Data, [hashtable] $Options)
 $Options = Get-InvoiceOptions $Options pdi
 $rows = @(Get-InvoiceRows $Data)
 & {
-    foreach ($head in ($rows | Select-Object -Property $Options.Head -Unique | Sort-Object $Options.Key)) {
-        Expand-InvoiceLine $Options.H $head $Options.Values
-        $lines = @($rows | Where-Object -Property $Options.Key -EQ -Value $head.($Options.Key))
-        foreach ($row in ($lines | Where-Object -Property $Options.Type -EQ -Value 'd')) { Expand-InvoiceLine $Options.D $row $Options.Values }
-        if ($Options.I) {
-            foreach ($row in ($lines | Where-Object -Property $Options.Type -EQ -Value 'i')) { Expand-InvoiceLine $Options.I $row $Options.Values }
+    foreach ($head in ($rows | Select-Object -Property $Options.Head -Unique)) {
+        if ($Options.H) { Expand-InvoiceLine $Options.H $head $Options.Values }
+        foreach ($row in ($rows | Where-Object -Property $Options.Key -EQ -Value $head.($Options.Key))) {
+            if (!$Options.Type -or $row.($Options.Type) -eq 'd') { Expand-InvoiceLine $Options.D $row $Options.Values }
+            elseif ($row.($Options.Type) -eq 'i' -and $Options.I) { Expand-InvoiceLine $Options.I $row $Options.Values }
         }
+        if ($Options.Footer) { Expand-InvoiceLine $Options.Footer $head $Options.Values }
     }
 } | Set-Content -LiteralPath $Options.Path -Encoding utf8NoBOM

DataAgent/fmt/text/frame.ps1

@@ -0,0 +1,27 @@
+# Text framing is after the formatter and before delivery, so adapters keep their normal write contract.
+. "$PSScriptRoot/../invoice/template.ps1"
+function Add-FormatFrame($Format, $Data, $Options) {
+    $scope = if ($Options.ContainsKey('Values')) { @{} + $Options['Values'] } else { @{} }
+    $scope.now = if ($scope.ContainsKey('now')) { [datetime]$scope.now } else { Get-Date }
+    $scope.sum = @{}
+    foreach ($key in $Format.Sum.Keys) {
+        $sum = $Format.Sum[$key]
+        $rows = $Data
+        if ($sum.UniqueBy) { $rows = $rows | Group-Object -Property $sum.UniqueBy | ForEach-Object { $_.Group[0] } }
+        $scope.sum[$key] = ($rows | ForEach-Object { $_.($sum.Property) -as [decimal] } | Measure-Object -Sum).Sum
+    }
+    $prefix = @(foreach ($text in $Format.Prepend) { if ($null -ne $text) { Expand-InvoiceLine $text $Data[0] $scope } })
+    $suffix = @(foreach ($text in $Format.Append) { if ($null -ne $text) { Expand-InvoiceLine $text $Data[0] $scope } })
+    if (!$prefix.Count -and !$suffix.Count) { return }
+    $encoding = if ($Options.Encoding) { $Options.Encoding } else { 'utf8NoBOM' }
+    foreach ($path in @($Options.Path)) {
+        if ([IO.Path]::GetExtension($path) -in '.xlsx', '.zip') { throw 'prepend/append applies to text, not XLSX or ZIP artifacts' }
+        $body = Get-Content -LiteralPath $path -Raw -Encoding $encoding
+        if ($suffix.Count) {
+            if ($body.Length -and !$body.EndsWith("`n") -and !$body.EndsWith("`r")) { $body += [Environment]::NewLine }
+            $body += ($suffix -join [Environment]::NewLine) + [Environment]::NewLine
+        }
+        if ($prefix.Count) { $body = ($prefix -join [Environment]::NewLine) + [Environment]::NewLine + $body }
+        Set-Content -LiteralPath $path -Value $body -Encoding $encoding -NoNewline
+    }
+}

DataAgent/fmt/x12-210.ps1

@@ -2,29 +2,53 @@ param($Data, [hashtable] $Options)
 . "$PSScriptRoot/invoice/template.ps1"
 $Options = Get-InvoiceOptions $Options 'x12-210'
 $rows = @(Get-InvoiceRows $Data)
-$now = Get-Date
+$now = $Options.Values.now
 $scope = @{} + $Options.Values
-$scope.now = $now
 # Kept from the extracted daily-job convention, not a unique interchange-number service.
 $scope.control = [int]('{0:00}{1:0000}' -f $now.ToString('yy'), ($now - [datetime]::new($now.Year, 1, 1)).TotalHours)
-foreach ($key in 'now', 'control') { if ($Options.Values.ContainsKey($key)) { $scope[$key] = $Options.Values[$key] } }
+$scope.day = $now.DayOfYear
+if ($Options.Control) { $scope.control = Expand-InvoiceLine $Options.Control $null $scope }
+if ($Options.Values.ContainsKey('control')) { $scope.control = $Options.Values.control }
+$scope.dates = @{}
+foreach ($key in $Options.Dates.Keys) { $scope.dates[$key] = $now.AddDays($Options.Dates[$key]) }
+
+function Expand-X12Segments($Templates, $Row) {
+    foreach ($entry in @($Templates)) {
+        $text = $entry
+        if ($entry -is [Collections.IDictionary]) {
+            if ($entry.When) {
+                $value = $Row.($entry.When.Field)
+                $match = if ($entry.When.ContainsKey('Is')) { [string]$value -eq [string]$entry.When.Is }
+                    else { ![string]::IsNullOrEmpty([string]$value) }
+                if ($entry.When.Not) { $match = !$match }
+                if (!$match) { continue }
+            }
+            $text = $entry.Text
+        }
+        foreach ($line in @($text)) {
+            (Expand-InvoiceLine ($line.Replace('*', $Options.Separator)) $Row $scope) + $Options.Terminator
+        }
+    }
+}
 & {
-    foreach ($t in $Options.Head) { Expand-InvoiceLine $t $null $scope }
+    Expand-X12Segments $Options.Head $null
     $st = 0
     foreach ($invoice in ($rows | Select-Object -Property * -ExcludeProperty $Options.Detail -Unique)) {
         $scope.st = ++$st
         $segments = [Collections.Generic.List[string]]::new()
-        foreach ($t in $Options.Invoice) { $segments.Add((Expand-InvoiceLine $t $invoice $scope)) }
+        foreach ($segment in (Expand-X12Segments $Options.Invoice $invoice)) { $segments.Add($segment) }
         $lx = 0
+        $detailCount = 0
         foreach ($line in ($rows | Where-Object -Property $Options.Key -EQ -Value $invoice.($Options.Key))) {
             $scope.lx = ++$lx
-            foreach ($t in $Options.Line) { $segments.Add((Expand-InvoiceLine $t $line $scope)) }
+            foreach ($segment in (Expand-X12Segments $Options.Line $line)) { $segments.Add($segment); $detailCount++ }
         }
-        foreach ($t in $Options.Close) { $segments.Add((Expand-InvoiceLine $t $invoice $scope)) }
+        foreach ($segment in (Expand-X12Segments $Options.Close $invoice)) { $segments.Add($segment) }
         $scope.se = $segments.Count + 1 + [int]$Options.SeExtra
-        $segments.Add((Expand-InvoiceLine $Options.Se $invoice $scope))
+        if ($null -ne $Options.SeDetailCount) { $scope.se += [int]$Options.SeDetailCount * $lx - $detailCount }
+        foreach ($segment in (Expand-X12Segments $Options.Se $invoice)) { $segments.Add($segment) }
         $segments
     }
     $scope.invoices = $st
-    foreach ($t in $Options.Tail) { Expand-InvoiceLine $t $null $scope }
+    Expand-X12Segments $Options.Tail $null
 } | Set-Content -LiteralPath $Options.Path -Encoding utf8NoBOM

README.md

@@ -3,13 +3,13 @@
 the repeated part of a feed job: get data, format it, send it. use the tools you already use.
 
 ```powershell
-Install-Module DataAgent -RequiredVersion 0.9.0 -Scope CurrentUser
+Install-Module DataAgent -RequiredVersion 0.10.0 -Scope CurrentUser
 ```
 
 ## the job
 
 ```powershell
-Import-Module DataAgent -RequiredVersion 0.9.0
+Import-Module DataAgent -RequiredVersion 0.10.0
 Invoke-DataAgent "$PSScriptRoot/settings.json"
 ```
 

examples/job.ps1

@@ -1,2 +1,2 @@
-Import-Module DataAgent -RequiredVersion 0.9.0 -ErrorAction Stop
+Import-Module DataAgent -RequiredVersion 0.10.0 -ErrorAction Stop
 Invoke-DataAgent "$PSScriptRoot/settings.json"

testing/DataAgent.Test/DataAgent.Test.psd1

@@ -1,6 +1,6 @@
 @{
     RootModule = 'DataAgent.Test.psm1'
-    ModuleVersion = '0.9.0'
+    ModuleVersion = '0.10.0'
     GUID = '3d16fcb0-a991-4c65-a6e4-3bfcd360688d'
     Author = 'Roy Ashbrook'
     Copyright = '(c) 2026 Roy Ashbrook. MIT License.'
@@ -10,7 +10,7 @@
     CmdletsToExport = @()
     VariablesToExport = @()
     AliasesToExport = @()
-    RequiredModules = @(@{ ModuleName = 'DataAgent'; RequiredVersion = '0.9.0' })
+    RequiredModules = @(@{ ModuleName = 'DataAgent'; RequiredVersion = '0.10.0' })
     FileList = @('DataAgent.Test.psd1', 'DataAgent.Test.psm1', 'LICENSE', 'synthetic.csv')
     PrivateData = @{ PSData = @{ Tags = @('ETL', 'DataAgent'); ProjectUri = 'https://github.com/royashbrook/DataAgent'; LicenseUri = 'https://github.com/royashbrook/DataAgent/blob/main/LICENSE' } }
 }

tests/acceptance.ps1

@@ -372,6 +372,7 @@ Remove-Item Env:DATAAGENT_TEST_SOURCE, Env:DATAAGENT_TEST_USER
 . "$PSScriptRoot/transfers.ps1"
 . "$PSScriptRoot/cosmos.ps1"
 & "$PSScriptRoot/invoice-formats.ps1"
+& "$PSScriptRoot/partner-options.ps1"
 
 foreach ($package in @('DataAgent','testing/DataAgent.Test')) {
     $name = Split-Path $package -Leaf
@@ -382,7 +383,7 @@ foreach ($package in @('DataAgent','testing/DataAgent.Test')) {
 }
 Remove-Module DataAgent -Force
 $env:PSModulePath = (@("$root/staged", $priorModules) -join [IO.Path]::PathSeparator)
-Import-Module DataAgent.Test -RequiredVersion 0.9.0
+Import-Module DataAgent.Test -RequiredVersion 0.10.0
 $result = @(Test-DataAgent)
 Assert (@($result | Where-Object { $_ -is [IO.FileInfo] -and $_.Name -eq 'output.csv' }).Count -eq 1) 'staged optional test package exercises bundled formatter'
 $env:PSModulePath = $priorModules

tests/invoice-formats.ps1

@@ -35,14 +35,18 @@ $null = $table.Rows.Add(@('A','20261014','3','i','','C','','','1','0','1','1','F
 $null = $table.Rows.Add(@('A','20261014','3','d','A1','C','DSL','','1','2','2','2','FRT'))
 $lines = Format pdi @($table.Rows) @{Values=@{sender_id='SYN'}}
 $expected = @(
-    '"H","SYN","A","20261014",,,,,,"3","N"'
-    '"D","A1","C","DSL","","SYN",,"1","2","2",,"2",'
-    '"I","FSC","1",'
     '"H","SYN","B","20261014",,,,,,"2","N"'
     '"D","B1","C","DSL","note, one","SYN",,"1","2","2",,"2",'
+    '"H","SYN","A","20261014",,,,,,"3","N"'
+    '"I","FSC","1",'
+    '"D","A1","C","DSL","","SYN",,"1","2","2",,"2",'
 )
 $expected | Set-Content "$root/expected.txt" -Encoding utf8NoBOM
-Check ((Get-FileHash "$root/pdi.txt").Hash -ceq (Get-FileHash "$root/expected.txt").Hash) 'pdi DataRow input has exact H/D/I bytes, sorted invoices and D before I'
+Check ((Get-FileHash "$root/pdi.txt").Hash -ceq (Get-FileHash "$root/expected.txt").Hash) 'pdi preserves first-seen invoice order and interleaved detail order byte for byte'
+$lines = Format pdi @($table.Rows[2], $table.Rows[1], $table.Rows[0]) @{Values=@{sender_id='SYN'}}
+Check (($lines -join '|') -ceq (($expected[2], $expected[4], $expected[3], $expected[0], $expected[1]) -join '|')) 'pdi already-sorted invoices and D-before-I input remain unchanged'
+$lines = Format pdi @($table.Rows) @{Values=@{sender_id='SYN'};I=''}
+Check (($lines -join '|') -ceq (($expected[0], $expected[1], $expected[2], $expected[4]) -join '|')) 'pdi can still omit I records without reordering the rest'
 $lines = Format pdi @([pscustomobject]@{id='A';kind='d';amount=2}) @{Key='id';Head='id';Type='kind';H='H,{id}';D='D,{amount:0.00}';I=''}
 Check (($lines -join '|') -ceq 'H,A|D,2.00') 'pdi record selectors and whole templates can be replaced'
 

tests/package.ps1

@@ -10,6 +10,7 @@ try {
     $installed = "$root/saved/DataAgent/$version"
     & "$PSScriptRoot/capabilities.ps1" -ModulePath $installed
     & "$PSScriptRoot/invoice-formats.ps1" -ModulePath $installed
+    & "$PSScriptRoot/partner-options.ps1" -ModulePath $installed
     if ((Get-FileHash "$installed/CAPABILITIES.md").Hash -ne (Get-FileHash "$PSScriptRoot/../DataAgent/CAPABILITIES.md").Hash) { throw 'packaged brief differs' }
     "PASS: published and saved package contains the exact brief ($version)"
 } finally {