diff --git a/go.mod b/go.mod index d84cae6bd..14709182e 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( charm.land/bubbles/v2 v2.2.1 charm.land/bubbletea/v2 v2.0.9 charm.land/lipgloss/v2 v2.0.6 - github.com/basecamp/basecamp-sdk/go v0.18.1-0.20260916210227-4523eac74cfa + github.com/basecamp/basecamp-sdk/go v0.19.0 github.com/basecamp/cli v0.2.2-0.20260828230226-767413fc712d github.com/basecamp/mcp v0.0.0-20260828100356-2d6f44b51e9d github.com/basecamp/surfguard/go v0.1.0 @@ -30,6 +30,7 @@ require ( golang.org/x/sys v0.48.0 golang.org/x/text v0.42.0 gopkg.in/yaml.v3 v3.0.1 + modernc.org/sqlite v1.59.0 ) require ( @@ -54,6 +55,7 @@ require ( github.com/charmbracelet/x/windows v0.2.2 // indirect github.com/clipperhouse/displaywidth v0.11.0 // indirect github.com/clipperhouse/uax29/v2 v2.7.0 // indirect + github.com/coder/websocket v1.8.15 // indirect github.com/cyberphone/json-canonicalization v0.0.0-20241213102144-19d51d7fe467 // indirect github.com/danieljoos/wincred v1.2.3 // indirect github.com/digitorus/pkcs7 v0.0.0-20230818184609-3a137a874352 // indirect @@ -99,7 +101,7 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/itchyny/timefmt-go v0.1.8 // indirect github.com/lucasb-eyer/go-colorful v1.4.1 // indirect - github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-isatty v0.0.24 // indirect github.com/mattn/go-localereader v0.0.1 // indirect github.com/mattn/go-runewidth v0.0.27 // indirect github.com/microcosm-cc/bluemonday v1.0.27 // indirect @@ -108,10 +110,12 @@ require ( github.com/muesli/cancelreader v0.2.2 // indirect github.com/muesli/reflow v0.3.0 // indirect github.com/muesli/termenv v0.16.0 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect github.com/oapi-codegen/runtime v1.7.0 // indirect github.com/oklog/ulid/v2 v2.1.1 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/secure-systems-lab/go-securesystemslib v0.11.0 // indirect github.com/segmentio/asm v1.1.3 // indirect @@ -144,4 +148,7 @@ require ( google.golang.org/grpc v1.83.2 // indirect google.golang.org/protobuf v1.36.11 // indirect k8s.io/klog/v2 v2.140.0 // indirect + modernc.org/libc v1.75.7 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.12.1 // indirect ) diff --git a/go.sum b/go.sum index bc9a9b162..389256aba 100644 --- a/go.sum +++ b/go.sum @@ -87,8 +87,8 @@ github.com/aymanbagabas/go-udiff v0.4.1 h1:OEIrQ8maEeDBXQDoGCbbTTXYJMYRCRO1fnodZ github.com/aymanbagabas/go-udiff v0.4.1/go.mod h1:0L9PGwj20lrtmEMeyw4WKJ/TMyDtvAoK9bf2u/mNo3w= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= -github.com/basecamp/basecamp-sdk/go v0.18.1-0.20260916210227-4523eac74cfa h1:efZJJSiwKn5lBih05oVfLM6S+ZYN6kB0vcH1DQmd8a8= -github.com/basecamp/basecamp-sdk/go v0.18.1-0.20260916210227-4523eac74cfa/go.mod h1:kIBDYwPMMD59PadNGxpH0YTQuI+blFPZ8MelGI0RK5Q= +github.com/basecamp/basecamp-sdk/go v0.19.0 h1:byygVVbJnWCZsyBNeAlztlUAV23ytLEhPx98WakNy+c= +github.com/basecamp/basecamp-sdk/go v0.19.0/go.mod h1:kIBDYwPMMD59PadNGxpH0YTQuI+blFPZ8MelGI0RK5Q= github.com/basecamp/cli v0.2.2-0.20260828230226-767413fc712d h1:jAzDrCCzDpIwhbFT1xVVs0z2xpXoDEkomHfKB2bUUp8= github.com/basecamp/cli v0.2.2-0.20260828230226-767413fc712d/go.mod h1:iTBTaWvsPEFIcZfkxQHEfISyJ6sZ7036K6bNx0RY3EE= github.com/basecamp/mcp v0.0.0-20260828100356-2d6f44b51e9d h1:zEQVGq1x1nhKMZ2TudFAcSJ32CHT8richI1vQakIKz4= @@ -148,6 +148,8 @@ github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJ github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb h1:EDmT6Q9Zs+SbUoc7Ik9EfrFqcylYqgPZ9ANSbTAntnE= github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb/go.mod h1:ZjrT6AXHbDs86ZSdt/osfBi5qfexBrKUdONk989Wnk4= +github.com/coder/websocket v1.8.15 h1:6B2JPeOGlpff2Uz6vOEH1Vzpi0iUz20A+lPVhPHtNUA= +github.com/coder/websocket v1.8.15/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg= github.com/coreos/go-oidc/v3 v3.17.0 h1:hWBGaQfbi0iVviX4ibC7bk8OKT5qNr4klBaCHVNvehc= github.com/coreos/go-oidc/v3 v3.17.0/go.mod h1:wqPbKFrVnE90vty060SB40FCJ8fTHTxSwyXJqZH+sI8= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= @@ -254,6 +256,8 @@ github.com/google/go-containerregistry v0.21.7 h1:/vPFuVXDjtFREsVArW+0h1CIl5urnO github.com/google/go-containerregistry v0.21.7/go.mod h1:kjSbt7/zMsKLWfnHrIvKvhXHUw91jbe9DNjPPJ32gXE= github.com/google/jsonschema-go v0.4.3 h1:/DBOLZTfDow7pe2GmaJNhltueGTtDKICi8V8p+DQPd0= github.com/google/jsonschema-go v0.4.3/go.mod h1:r5quNTdLOYEz95Ru18zA0ydNbBuYoo9tgaYcxEYhJVE= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk= github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/trillian v1.7.3 h1:hziW+vo4czis48tzx2GK5xRBl/ZxBA9B0/UR5avXOro= @@ -286,6 +290,8 @@ github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 h1:kes8mmyCpxJsI7FTwtzRqEy9 github.com/hashicorp/go-secure-stdlib/strutil v0.1.2/go.mod h1:Gou2R9+il93BqX25LAKCLuM+y9U2T4hlwvT1yprcna4= github.com/hashicorp/go-sockaddr v1.0.7 h1:G+pTkSO01HpR5qCxg7lxfsFEZaG+C0VssTy/9dbT+Fw= github.com/hashicorp/go-sockaddr v1.0.7/go.mod h1:FZQbEYa1pxkQ7WLpyXJ6cbjpT8q0YgQaK/JakXqGyWw= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.1-vault-7 h1:ag5OxFVy3QYTFTJODRzTKVZ6xvdfLLCA1cy/Y6xGI0I= github.com/hashicorp/hcl v1.0.1-vault-7/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= github.com/hashicorp/vault/api v1.22.0 h1:+HYFquE35/B74fHoIeXlZIP2YADVboaPjaSicHEZiH0= @@ -319,8 +325,8 @@ github.com/letsencrypt/boulder v0.20260309.0 h1:kZynrxK3QfqLGx6hhoz+Rfs3hgltJs1p github.com/letsencrypt/boulder v0.20260309.0/go.mod h1:yG8lj8pNPZ8taq3oNdTpfBS+eC74IaEuiewqzVpXiWE= github.com/lucasb-eyer/go-colorful v1.4.1 h1:1EO+WB73+EH8EVbzlrG3KLAfEypQWVHIBqlTf+2hNss= github.com/lucasb-eyer/go-colorful v1.4.1/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= -github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= -github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= +github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4= github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88= github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= @@ -346,6 +352,8 @@ github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= github.com/natefinch/atomic v1.0.1 h1:ZPYKxkqQOx3KZ+RsbnP/YsgvxWQPGxjC0oBt2AhwV0A= github.com/natefinch/atomic v1.0.1/go.mod h1:N/D/ELrljoqDyT3rZrsUmtsuzvHkeB/wWjHV22AZRbM= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/oapi-codegen/nullable v1.1.0 h1:eAh8JVc5430VtYVnq00Hrbpag9PFRGWLjxR1/3KntMs= github.com/oapi-codegen/nullable v1.1.0/go.mod h1:KUZ3vUzkmEKY90ksAmit2+5juDIhIZhfDl+0PwOQlFY= github.com/oapi-codegen/runtime v1.7.0 h1:t7358VYPvNbWJ9gdAkIK/smVeHpBf6yp8VTsaZsb/7k= @@ -360,6 +368,8 @@ github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjL github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= @@ -487,7 +497,6 @@ golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7 golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE= @@ -519,6 +528,34 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0= +modernc.org/cc/v4 v4.29.2 h1:h6+9ciCnPKutf4I03CvheAvDLX7+IHlqR6Iy6J+cgd8= +modernc.org/cc/v4 v4.29.2/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.35.0 h1:F+TUsmw09QxLzmi3aeYYGxjAXarmZaKgj3mKQHNaA8w= +modernc.org/ccgo/v4 v4.35.0/go.mod h1:qrVGs9S3Sr2Ztcg9ve+kTAYMp5a3YvWjo+SoN06kJ5I= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.5 h1:21ldfPfRYE31Tb7B3mwAK8gy1AxP4+dKjrOQPfqakoc= +modernc.org/gc/v3 v3.1.5/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.75.7 h1:o3DTP9/0p9pKmY2WCKQaySW6wIiZhNM7wc2lUoyhfew= +modernc.org/libc v1.75.7/go.mod h1:bO5o2ztHxBb2rjz0PgdHN0sSMw57CgxGFLZ3Qd/QpVQ= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.12.1 h1:nFMiWrpStgZczNl6XI9GnIk/rWhYIyHGUaR04pGbp9g= +modernc.org/memory v1.12.1/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= +modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.59.0 h1:X1es1GpqBlS/5T+vbM4HLUdaa8OtQx468DF2vrx+38A= +modernc.org/sqlite v1.59.0/go.mod h1:+paeT2A3iPRHkQDwG7oA6Tk0zQd5woMEI8q7orfry8k= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= software.sslmate.com/src/go-pkcs12 v0.4.0 h1:H2g08FrTvSFKUj+D309j1DPfk5APnIdAQAB8aEykJ5k= diff --git a/internal/connector/admission/commit_test.go b/internal/connector/admission/commit_test.go index 731f67a27..919cdca0d 100644 --- a/internal/connector/admission/commit_test.go +++ b/internal/connector/admission/commit_test.go @@ -1,6 +1,8 @@ package admission import ( + "bufio" + "bytes" "context" "encoding/json" "errors" @@ -396,7 +398,7 @@ func (stuckWriter) Write([]byte) (int, error) { return 0, nil } func TestALineIsWrittenWholeOrNotAtAll(t *testing.T) { w := &chunkWriter{chunk: 7} - require.NoError(t, (&lineWriter{w: w}).write(Verdict{EventID: 1, EventType: "card.created", State: StateDiscarded, Reason: ReasonNotInMatrix})) + require.NoError(t, newLineWriter(nil, w).write(Verdict{EventID: 1, EventType: "card.created", State: StateDiscarded, Reason: ReasonNotInMatrix})) var m map[string]any require.NoError(t, json.Unmarshal([]byte(strings.TrimSuffix(w.b.String(), "\n")), &m), "a writer that takes a line in pieces still gets all of it") assert.True(t, strings.HasSuffix(w.b.String(), "}\n")) @@ -404,7 +406,7 @@ func TestALineIsWrittenWholeOrNotAtAll(t *testing.T) { // Bounded: a writer loop that forgot this case would spin, and should fail // this test by name rather than hang the suite. done := make(chan error, 1) - go func() { done <- (&lineWriter{w: stuckWriter{}}).write(Verdict{EventID: 1, State: StateDiscarded}) }() + go func() { done <- newLineWriter(nil, stuckWriter{}).write(Verdict{EventID: 1, State: StateDiscarded}) }() select { case err := <-done: require.ErrorIs(t, err, io.ErrShortWrite) @@ -416,7 +418,7 @@ func TestALineIsWrittenWholeOrNotAtAll(t *testing.T) { func TestLinesCannotCarryTerminalControls(t *testing.T) { esc, csi, bel := string(rune(0x1b)), string(rune(0x9b)), string(rune(0x07)) var b strings.Builder - require.NoError(t, (&lineWriter{w: &b}).write(Verdict{ + require.NoError(t, newLineWriter(nil, &b).write(Verdict{ EventID: 1, EventType: "card.created" + esc + "]0;owned" + bel, RecordingURL: "https://app.basecamp.com/x" + csi + "31m" + esc + "[2J", @@ -462,9 +464,65 @@ func TestRunStopsOnALoadFailure(t *testing.T) { func TestLinesKeepLocalPathsAsWritten(t *testing.T) { var b strings.Builder - require.NoError(t, (&lineWriter{w: &b}).write(Verdict{EventID: 1, State: StateAdmitted, Route: " /work/My Projects\tA", Class: "in ternal"})) + require.NoError(t, newLineWriter(nil, &b).write(Verdict{EventID: 1, State: StateAdmitted, Route: " /work/My Projects\tA", Class: "in ternal"})) var m map[string]any require.NoError(t, json.Unmarshal([]byte(b.String()), &m)) assert.Equal(t, " /work/My Projects\tA", m["route"], "whitespace in a path is not a terminal control") assert.Equal(t, "in ternal", m["class"]) } + +// A sink that is safe for concurrent use, so the only race a run can report is +// one of admission's own making. +type lockedSink struct { + mu sync.Mutex + buf bytes.Buffer +} + +func (s *lockedSink) Write(p []byte) (int, error) { + s.mu.Lock() + defer s.mu.Unlock() + return s.buf.Write(p) +} + +// The verdict writer is built before any worker exists. Built on first use, a +// pool of workers reaching their first lines together would read and assign it +// at once — a data race on the ordinary path, not an edge case. +func TestTheVerdictWriterIsBuiltBeforeAnyWorkerWrites(t *testing.T) { + sink := &lockedSink{} + lines := newLineWriter(nil, sink) + + var wg sync.WaitGroup + for worker := range 8 { + wg.Go(func() { + for i := range 25 { + require.NoError(t, lines.write(Verdict{ + EventID: int64(worker*100 + i), + EventType: "comment.created", + State: StateAdmitted, + })) + } + }) + } + wg.Wait() + + scanner := bufio.NewScanner(bytes.NewReader(sink.buf.Bytes())) + count := 0 + for scanner.Scan() { + var line map[string]any + require.NoError(t, json.Unmarshal(scanner.Bytes(), &line), "torn line %d: %q", count, scanner.Text()) + count++ + } + require.NoError(t, scanner.Err()) + assert.Equal(t, 200, count) +} + +// One sink has one writer, whoever asks for it: the lock that keeps lines +// whole is only one lock if there is only one of it. +func TestOneSinkHasOneWriter(t *testing.T) { + sink := &lockedSink{} + + first := newLineWriter(nil, sink) + second := newLineWriter(nil, sink) + + assert.Same(t, first.out, second.out) +} diff --git a/internal/connector/admission/run.go b/internal/connector/admission/run.go index df5700932..421d94f02 100644 --- a/internal/connector/admission/run.go +++ b/internal/connector/admission/run.go @@ -2,13 +2,13 @@ package admission import ( "context" - "encoding/json" "errors" "fmt" "io" "log/slog" "sync" + "github.com/basecamp/basecamp-cli/internal/connector/ndjson" "github.com/basecamp/basecamp-cli/internal/richtext" ) @@ -42,8 +42,12 @@ type RunOptions struct { // Workers is the fetcher pool size; DefaultWorkers when zero. Workers int // Lines receives one NDJSON line per committed verdict. - Lines io.Writer - Logger *slog.Logger + Lines io.Writer + // LineWriter, when set, is the writer the lines go through, and Lines is + // ignored: one sink has one writer and one lock, and a caller wiring + // admission beside intake passes the same writer to both. + LineWriter *ndjson.Writer + Logger *slog.Logger } // Run takes ids until ctx ends, deciding and committing each. It returns nil @@ -68,7 +72,9 @@ func Run(ctx context.Context, opts RunOptions) error { if log == nil { log = slog.New(slog.DiscardHandler) } - lines := &lineWriter{w: opts.Lines} + // Built here, before the workers exist, so nothing is initialized on a + // path two of them can take at once. + lines := newLineWriter(opts.LineWriter, opts.Lines) ctx, cancel := context.WithCancel(ctx) defer cancel() @@ -178,33 +184,35 @@ func LineFor(v Verdict) Line { } } +// lineWriter is the verdict stream. It holds a writer and nothing else: no +// flag, no once, no first-use path. +// +// It used to build the writer on the first verdict, which is a race the moment +// admission has more than one worker — two of them reaching their first line +// together read and assign the same field. Laziness is not a thing to guard +// here; it is a thing to remove. The writer is built once, before any worker +// starts, and injected. type lineWriter struct { - mu sync.Mutex - w io.Writer + out *ndjson.Writer +} + +// newLineWriter takes the writer a caller wiring admission beside intake +// passes in, or builds one for the sink. There is exactly one writer per sink +// either way: NewWriter returns the same writer for the same sink, so the one +// lock is the one lock. +func newLineWriter(out *ndjson.Writer, sink io.Writer) *lineWriter { + if out == nil && sink != nil { + out = ndjson.NewWriter(sink) + } + return &lineWriter{out: out} } func (l *lineWriter) write(v Verdict) error { - if l.w == nil { + if l.out == nil { return nil } - b, err := json.Marshal(LineFor(v)) - if err != nil { - return fmt.Errorf("admission: encode line: %w", err) - } - l.mu.Lock() - defer l.mu.Unlock() - // One line, whole: a writer that takes part of it is written the rest, - // and one that takes none without an error has failed. A torn line is - // worse than no line to whoever parses the stream. - for rest := append(b, '\n'); len(rest) > 0; { - n, err := l.w.Write(rest) - if err != nil { - return fmt.Errorf("admission: write line: %w", err) - } - if n <= 0 { - return fmt.Errorf("admission: write line: %w", io.ErrShortWrite) - } - rest = rest[n:] + if err := l.out.WriteLine(LineFor(v)); err != nil { + return fmt.Errorf("admission: %w", err) } return nil } diff --git a/internal/connector/doc.go b/internal/connector/doc.go new file mode 100644 index 000000000..127aa1927 --- /dev/null +++ b/internal/connector/doc.go @@ -0,0 +1,53 @@ +// Package connector is the intake half of `basecamp connect`: the account +// event feed, the durable ledger behind it, and the recovery that makes a +// crash a delay rather than a loss. +// +// # What intake is, and what it deliberately is not +// +// Feed rows are pointers — id, type, bucket, creator, recording — and nothing +// else. No title, no body, no URL, no names. Intake writes that pointer to the +// ledger if the id is new and hands the id to a queue. That is the whole of +// the work on the feed's delivery path: a slow admission or a busy dispatcher +// is absorbed by the backlog rather than felt by the socket. +// +// Absorbed, not unbounded. The backlog is a bounded queue, and at its pause +// threshold an offer waits for room — which is intake deliberately stopping +// its read of the feed, so the backlog cannot grow until it is the process's +// memory that fails. The decoupling holds up to that depth; past it, +// backpressure is the design, and the pause is reported rather than hidden. +// +// Deciding whether an event deserves an agent's attention is admission's job +// and it costs a read per event it cares about. Intake does none of it. It +// also does no filtering of its own: the feed carries no `column_id` or +// `todolist_id`, so narrowing to a column or a list means re-fetching the +// parent listing, and that price belongs where the read already happens. +// +// # The two lanes +// +// The live lane is a WebSocket; the poll lane is an HTTP walk that runs about +// thirty seconds behind it, deliberately, so a page never stops mid-commit. +// The same event therefore arrives twice in the ordinary case, and dedupe is +// by event id across both lanes and across restarts — which is what makes the +// ledger, not a set in memory, the dedupe authority. +// +// Only the poll lane advances the durable position. A live event id is far +// ahead of the poll lane, and a checkpoint taken from one would skip +// everything the safety delay had not yet served. +// +// # Quiet is not caught up +// +// Three separate things in this feed look like "nothing more to read" and are +// not: +// +// - An empty page. A request crosses up to a thousand ledger rows and serves +// at most a hundred matches; rows the filters exclude still advance the +// cursor. Follow `next` until it is absent, never stop at the first empty +// page. +// - A missing `next`. The walk reached its frozen head, which is not the end +// of history: a page cut short by the safety horizon withholds the link on +// purpose. Poll again later. +// - A silent inbox. Delivery has write-time brakes — a per-account rate +// ceiling and an agent-to-agent chain breaker — and a braked event writes +// no addressing and tells the client nothing. A quiet feed is not proof of +// a quiet project, so nothing here ever reports "up to date". +package connector diff --git a/internal/connector/intake.go b/internal/connector/intake.go new file mode 100644 index 000000000..e47d2e5b3 --- /dev/null +++ b/internal/connector/intake.go @@ -0,0 +1,1398 @@ +package connector + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + "net/url" + "slices" + "strconv" + "sync" + "time" + + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" + + "github.com/basecamp/basecamp-cli/internal/connector/ndjson" + "github.com/basecamp/basecamp-cli/internal/richtext" +) + +// Defaults for the recovery timers. +const ( + // DefaultRepairInterval is how often a repair walk re-runs while a loss is + // open. It matches the feed's own repair poll, and it is deliberately + // longer than the poll lane's ~30s safety delay: a walk that outran the + // delay would call a still-committing event missing. + DefaultRepairInterval = 60 * time.Second + // DefaultRepairWindow is how long a loss stays open before the ids still + // missing are called unrecovered. At the sixty-second cadence that is + // about ten repair polls, each well past the safety delay. + DefaultRepairWindow = 10 * time.Minute + // DefaultMembershipInterval is how often the agent's project list is + // re-read. The cable snapshots the agent's buckets when it subscribes, so + // a project granted afterwards is invisible on the live lane until the + // connection is remade. + DefaultMembershipInterval = 10 * time.Minute +) + +// MembershipSource lists the buckets the agent can currently see. +type MembershipSource interface { + Buckets(ctx context.Context) ([]int64, error) +} + +// Options configures intake. +type Options struct { + // Origin is the API base URL. It is part of the checkpoint identity and + // the origin every continuation URL is validated against. + Origin string + // AccountID is the Basecamp account. + AccountID string + // ConsumerNamespace names this connector's checkpoint lineage. Two + // connectors in one account must not share one. + ConsumerNamespace string + // Filters is the feed's filter set. The checkpoint is keyed by its digest, + // so changing it re-enters under a new lineage. + Filters eventfeed.Filters + // SinceEventID, when positive, enters just after that event id whatever + // the ledger holds — the `--since` override. + SinceEventID int64 + + Ledger *Ledger + Queue *Queue + Minter eventfeed.TicketMinter + // PollsFor returns a fresh poll source per walk. The SDK's live source + // holds walk state (the order a continuation must follow), so the feed's + // connection and each repair walk get their own. Polls, when set instead, + // is shared by all of them — for test fakes that hold no walk state. + PollsFor func() eventfeed.PollSource + Polls eventfeed.PollSource + + // Pointers receives one NDJSON line per newly seen event. Writes are + // serialized: an interleaved write tears a line and breaks the watcher + // reading it. + Pointers io.Writer + // Lines, when set, is the writer the pointer lines go through, and + // Pointers is ignored. One process stdout has one writer and one lock, so + // a caller wiring intake and admission onto the same stdout passes the + // same writer to both rather than relying on them to find each other. + Lines *ndjson.Writer + // Logger receives everything else. Pointer lines are the protocol; + // logging is not. + Logger *slog.Logger + + // Membership, when set, drives the reconnect that makes a newly granted + // project visible on the live lane. + Membership MembershipSource + MembershipInterval time.Duration + + RepairInterval time.Duration + RepairWindow time.Duration + + Clock func() time.Time + Transport eventfeed.CableTransport +} + +// LiveOptions binds intake to the live feed: the SDK's own seams over the +// generated operations, with their redirect guard, their two-410 mapping and +// their continuation checks. The caller fills in the rest (account, namespace, +// ledger, queue, filters). +// +// Build the Live without a debug logger or request hooks in production: the +// SDK client logs request URLs through them, and a poll URL carries the feed +// position. +func LiveOptions(live *eventfeed.Live) Options { + return Options{ + Origin: live.Origin(), + Minter: live.Minter(), + PollsFor: live.Polls, + } +} + +// Intake is the feed's delivery path: write the pointer, hand over the id. +// +// Everything else — reading the recording, judging it, dispatching it — is +// downstream of the queue, so a slow admission or a busy dispatcher is +// absorbed by the backlog rather than felt by the socket — until the backlog +// reaches its pause threshold, where intake stops reading the feed on purpose +// rather than let the queue grow without bound. +type Intake struct { + opts Options + ledger *Ledger + queue *Queue + log *slog.Logger + now func() time.Time + pointer *pointerWriter + + key eventfeed.CheckpointKey + // positions reads the safe re-entry ids; the ledger, except in tests that + // need the read to fail. + positions pollServedReader + + mu sync.Mutex + // served is the current connection's wrapped poll source. + served *servedPolls + // listed is what the project lister last said, and it is authoritative: a + // bucket it stops naming is revoked. + listed map[int64]bool + // membershipRetry is the first backoff after a failed membership read. + membershipRetry time.Duration + // learned holds buckets an arriving event proved visible that the lister + // has not named. It is provisional — the next listing that omits one takes + // it back — so a bucket learned at runtime never becomes permanent truth. + learned map[int64]bool + // relearned is when each bucket was last learned. A revoked bucket is + // learned again at most once per membership interval, so revocation + // cannot become a reconnect per event. + relearned map[int64]time.Time + reconnect chan struct{} + // cancelRun ends the current connection; nil between connections. + cancelRun context.CancelFunc + // promotedThisRun is whether this connection has recorded a poll-served + // id. Until it has, the package's own reset cursor is zero, and a refused + // position would send it to the present. + promotedThisRun bool + // reentry is the safe entry the next connection takes after a refused + // stored position; hasReentry says whether one is pending. + reentry eventfeed.Start + hasReentry bool + // replaying is whether this connection entered at the beginning of served + // history on purpose, to recover. + replaying bool + // reentryReplays is whether the pending re-entry is a replay. + reentryReplays bool + // checkpointed is whether any connection in this run has saved a + // position. + checkpointed bool + // enteredByReentry is whether this connection is itself the safe + // re-entry after a refused position. + enteredByReentry bool + reentryLog string + // abortErr ends the run: set when continuing could only mean entering + // the feed somewhere unsafe. + abortErr error + + // stranded holds ids this process committed to the ledger and then failed + // to hand over. They are the one thing the ledger's own dedupe would hide: + // the row is there, so every retry of the event is suppressed as a + // duplicate, and without this the id would wait for a restart. + stranded map[int64]struct{} + + repairs sync.WaitGroup + repairQueue chan Loss + // repairQueueSize and repairSweep override the pool's defaults in tests. + repairQueueSize int + repairSweep time.Duration + // inFlight is the losses a worker is walking or the queue is holding, so + // the sweeper does not offer one twice. + inFlight map[int64]bool + // lifetime is Run's context. Repair walks are bound to it rather than to + // a connection, so a reconnect does not abandon a walk and a shutdown does + // not strand Run waiting on one — an unfinished walk simply resumes on the + // next start, which is what OpenLosses is for. + lifetime context.Context + // repairSleep replaces the wait between repair polls. Tests set it so ten + // minutes of repair cadence does not take ten minutes. + repairSleep func(ctx context.Context, d time.Duration) error +} + +// New builds intake. +func New(opts Options) (*Intake, error) { + switch { + case opts.Ledger == nil: + return nil, errors.New("connector: intake needs a ledger") + case opts.Queue == nil: + return nil, errors.New("connector: intake needs a queue") + case opts.Minter == nil || (opts.Polls == nil && opts.PollsFor == nil): + return nil, errors.New("connector: intake needs the feed's two seams") + case opts.AccountID == "": + return nil, errors.New("connector: intake needs an account id") + case opts.ConsumerNamespace == "": + return nil, errors.New("connector: intake needs a consumer namespace") + } + + if opts.PollsFor == nil { + shared := opts.Polls + opts.PollsFor = func() eventfeed.PollSource { return shared } + } + + origin, err := eventfeed.CanonicalOrigin(opts.Origin) + if err != nil { + return nil, fmt.Errorf("connector: intake origin: %w", err) + } + if err := opts.Filters.Validate(); err != nil { + return nil, fmt.Errorf("connector: intake filters: %w", err) + } + if len(opts.Filters.Reasons) > 0 { + // Reasons is the inbox lane's dimension — why an event reached YOU — + // and the account feed does not carry it. The SDK refuses it when the + // feed is built, but that is too late: Run starts the repair workers + // before the feed exists, so an open loss could walk under a filter + // set the account lane cannot honor, and a dropped dimension widens a + // read rather than narrowing it. Configuration that cannot mean what + // it says fails here, before any wire work. + return nil, errors.New("connector: intake filters: reasons filter the inbox, which the account feed does not carry") + } + // The filter set is the checkpoint's identity, and it is also what every + // subscription, recorded loss and repair walk runs under. A caller that + // kept its slices could change all of those while the key stays frozen on + // what it was built from, so they are copied here as the SDK's WithFilters + // copies its own. + opts.Filters = eventfeed.Filters{ + Types: slices.Clone(opts.Filters.Types), + Buckets: slices.Clone(opts.Filters.Buckets), + Creators: slices.Clone(opts.Filters.Creators), + Performers: slices.Clone(opts.Filters.Performers), + ExcludePerformers: slices.Clone(opts.Filters.ExcludePerformers), + ActorTypes: slices.Clone(opts.Filters.ActorTypes), + Reasons: slices.Clone(opts.Filters.Reasons), + } + + if opts.Clock == nil { + opts.Clock = time.Now + } + if opts.Logger == nil { + opts.Logger = slog.New(slog.DiscardHandler) + } + // The queue reports a callback that panicked, and a queue built by a + // caller who did not think about that would report it nowhere. It is + // adopted under the queue's own lock, because the queue may already be + // in use — admission takes from it — and a logger the caller did set is + // kept. + opts.Queue.adoptLogger(opts.Logger) + if opts.RepairInterval <= 0 { + opts.RepairInterval = DefaultRepairInterval + } + if opts.RepairWindow <= 0 { + opts.RepairWindow = DefaultRepairWindow + } + if opts.MembershipInterval <= 0 { + opts.MembershipInterval = DefaultMembershipInterval + } + + in := &Intake{ + opts: opts, + ledger: opts.Ledger, + queue: opts.Queue, + log: opts.Logger, + now: opts.Clock, + pointer: newPointerWriter(opts.Pointers, opts.Lines), + reconnect: make(chan struct{}, 1), + key: eventfeed.CheckpointKey{ + Origin: origin, + AccountID: opts.AccountID, + ConsumerNamespace: opts.ConsumerNamespace, + FilterKey: opts.Filters.FilterKey(), + }, + } + in.ledger.now = opts.Clock + in.positions = opts.Ledger + return in, nil +} + +// pollServedReader is the slice of the ledger re-entry reads. +type pollServedReader interface { + LastPollServedID(ctx context.Context, key eventfeed.CheckpointKey) (int64, error) + LineagePollServedID(ctx context.Context, key eventfeed.CheckpointKey) (int64, error) +} + +// CheckpointKey is the identity this intake's position is stored under. +func (in *Intake) CheckpointKey() eventfeed.CheckpointKey { return in.key } + +// Run consumes the feed until ctx is canceled or the feed terminates. +// +// It reconnects on its own for two reasons only — membership, and a stored +// position refused before this run had a safe re-entry of its own. Everything +// else the feed can recover from, it recovers from inside the package. +func (in *Intake) Run(ctx context.Context) error { + in.resetRunState() + // Repairs get a child lifetime that ends when Run does, whatever the + // reason. A repair is off the delivery path: a terminal feed must not wait + // out its sixty-second cadence, and an unfinished repair resumes on the + // next start from the loss record. + repairCtx, stopRepairs := context.WithCancel(ctx) + // Deferred first, so it runs last: the pool is forgotten only once its + // workers have stopped, and the next Run starts its own. + defer in.releaseRepairWorkers() + defer in.repairs.Wait() + defer stopRepairs() + in.lifetime = repairCtx + + if err := in.resumeReconciliation(repairCtx); err != nil { + return err + } + if err := in.requeueSeen(ctx); err != nil { + return err + } + + since := in.opts.SinceEventID + for { + err := in.runOnce(ctx, since) + // --since is an entry, not a standing instruction: once a checkpoint + // has been saved, a reconnect resumes from it. Until then the explicit + // entry is the only position this run has, and a connection that ends + // before its first save must not leave the next one entering at the + // present. + if in.checkpointSaved() { + since = 0 + } + if ctx.Err() != nil { + return ctx.Err() + } + switch { + case errors.Is(err, errReconnect): + in.log.Info("reconnecting the feed") + default: + return err + } + } +} + +// resetRunState clears everything scoped to one Run. +// +// Run is reusable — each one builds its own repair pool — so nothing a +// previous one decided may leak into the next: a checkpoint it saved would +// clear this run's --since before this run has saved anything, and an abort it +// suffered would end this one before it began. What survives is knowledge +// about the account rather than the run: the listed and learned projects. +func (in *Intake) resetRunState() { + in.mu.Lock() + defer in.mu.Unlock() + in.checkpointed = false + in.abortErr = nil + in.hasReentry = false + in.reentry = eventfeed.Start{} + in.reentryLog = "" + in.reentryReplays = false + in.replaying = false + in.enteredByReentry = false + in.promotedThisRun = false + select { + case <-in.reconnect: + default: + } +} + +// errReconnect asks the supervisor for a fresh connection. It is not a +// failure. +var errReconnect = errors.New("connector: reconnect the feed") + +func (in *Intake) runOnce(ctx context.Context, since int64) error { + runCtx, cancel := context.WithCancel(ctx) + defer cancel() + + // This connection IS the reconnect any request made between connections + // asked for; a latch left standing would turn its first terminal error + // into a silent reconnect. + in.reconnectRequested() + + in.mu.Lock() + in.cancelRun = cancel + in.promotedThisRun = false + reentry, hasReentry, reentryLog, reentryReplays := in.reentry, in.hasReentry, in.reentryLog, in.reentryReplays + in.hasReentry = false + in.replaying = false + in.enteredByReentry = hasReentry + in.mu.Unlock() + defer func() { + in.mu.Lock() + in.cancelRun = nil + in.mu.Unlock() + }() + + in.takeSnapshot(runCtx) + + _, hadPosition, err := in.ledger.Load(runCtx, in.key) + if err != nil { + return err + } + ownServed, err := in.ledger.LastPollServedID(runCtx, in.key) + if err != nil { + return err + } + lineageServed, err := in.ledger.LineagePollServedID(runCtx, in.key) + if err != nil { + return err + } + lineageHasPosition, err := in.ledger.LineageHasPosition(runCtx, in.key) + if err != nil { + return err + } + + start := eventfeed.StartResume() + switch { + case since > 0: + start = eventfeed.StartAfter(since) + in.log.Info("entering the feed after an explicit event id", "since", since) + case hasReentry: + start = reentry + in.setReplaying(reentryReplays) + in.log.Warn("the stored position was refused; " + reentryLog) + case hadPosition: + in.log.Info("resuming the feed from the stored position", "filter_key", in.key.FilterKey) + case ownServed > 0: + // This filter set has progress but no position: the package confirms + // a page before it saves the page's position, so a failed save leaves + // exactly this. Its own id, never another filter set's. + start = eventfeed.StartAfter(ownServed) + in.log.Warn("no stored position for this filter set; re-entering after its own last poll-served id", + "since", ownServed, "filter_key", in.key.FilterKey) + case lineageServed > 0: + // A filter change: this digest has no position, but the consumer's + // poll lane had reached this id under another. Entering at the + // present would skip everything since. + start = eventfeed.StartAfter(lineageServed) + in.log.Warn("no position for this filter set; re-entering after the last poll-served id under the previous one", + "since", lineageServed, "filter_key", in.key.FilterKey) + case lineageHasPosition: + // A filter change from a set that had read up to a position but + // recorded no served id. Entering at the present would skip what + // followed that position; replaying from the beginning costs reads + // the ledger absorbs. + start = eventfeed.StartBeginning() + in.setReplaying(true) + in.log.Warn("no position or poll-served id for this filter set, but the previous one had read up to a position; replaying from the beginning of served history", + "filter_key", in.key.FilterKey) + default: + // Said out loud because it is a real loss of history, not a neutral + // default: everything committed before this moment is never served. + in.log.Warn("no stored position: entering the feed at the present, so nothing committed before now will be served", + "filter_key", in.key.FilterKey) + } + + options := []eventfeed.Option{ + eventfeed.WithFilters(in.opts.Filters), + eventfeed.WithStart(start), + eventfeed.WithCheckpointStore(in.ledger), + eventfeed.WithConsumerNamespace(in.opts.ConsumerNamespace), + eventfeed.WithSignalHandler(in.handleSignal), + eventfeed.WithObserver(in.observer(runCtx)), + eventfeed.WithRepairInterval(in.opts.RepairInterval), + } + if in.opts.Transport != nil { + options = append(options, eventfeed.WithTransport(in.opts.Transport)) + } + + served := &servedPolls{inner: in.opts.PollsFor()} + in.mu.Lock() + in.served = served + in.mu.Unlock() + + feed, err := eventfeed.New(in.key.Origin, in.opts.AccountID, in.opts.Minter, served, options...) + if err != nil { + return fmt.Errorf("connector: build feed: %w", err) + } + // Deferred in this order so they run in the reverse: the connection is + // canceled first, which is what lets the membership watcher and the feed + // return, and only then are they waited on. + defer func() { + _ = feed.Close() + feed.Wait() + }() + // The watcher's context is created and canceled here, in its owner, so + // the cancel is provably reached on every path out of this connection. + watchCtx, stopWatch := context.WithCancel(runCtx) + defer stopWatch() + awaitMembership := in.watchMembership(watchCtx) + defer func() { + stopWatch() + awaitMembership() + }() + defer cancel() + + var feedErr error + for event, err := range feed.Events(runCtx) { + if err != nil { + feedErr = err + break + } + // Run's context, not the connection's: a reconnect canceling the + // connection while this hand-off waits for queue room would leave the + // event seen in the ledger and never queued, since the re-served page + // no longer reports it new. The reconnect waits for the hand-off; the + // feed stays paused either way. + if err := in.ingest(ctx, event, LaneOf(event)); err != nil { + if in.reconnectRequested() { + return errReconnect + } + return err + } + } + + in.mu.Lock() + aborted := in.abortErr + in.mu.Unlock() + if aborted != nil { + return aborted + } + if in.reconnectRequested() { + return errReconnect + } + // The two 410s are told apart below this package: the SDK's live seam + // maps the feed's FeedPositionGoneError (epoch required) to the gap + // signal, and refuses the inbox's InboxPositionGoneError shape on the + // account lane as unrecoverable, so it ends the feed here as a terminal + // rather than resuming as if it were the epoch's. + return feedErr +} + +// ingest is the whole of intake: one pointer written, one id handed over. +func (in *Intake) ingest(ctx context.Context, event eventfeed.Event, lane Lane) error { + fresh, err := in.ledger.RecordSeen(ctx, event, lane) + if err != nil { + // A pointer that could not be written is a pointer the restart will + // not know about. Nothing downstream is allowed to proceed as if it + // had been. + return err + } + if !fresh { + // The ordinary case: the poll lane serving what the live lane already + // delivered, or a restart re-walking a page. Dedupe is the point. + return nil + } + + // Past this line the row is committed, so the event is invisible to every + // later delivery of itself: it is this process's to finish handing over, + // or to remember that it did not. Both failures below are that. + if err := in.pointer.write(event, lane); err != nil { + in.strand(event.ID) + return err + } + if err := in.queue.Offer(ctx, event.ID); err != nil { + in.strand(event.ID) + return err + } + // Only once the id is handed over: the reconnect cancels the connection + // this event arrived on, and the event must not be stranded between the + // ledger and the queue by it. + in.noteBucket(event.BucketID) + return nil +} + +// LaneOf says which lane served an event. +// +// The rows are not labeled, but the two shapes differ: actor_type and +// visible_to_clients are push-lane transport fields that poll rows omit, and +// the SDK keeps both presence-bearing — a string whose empty value is outside +// the vocabulary, and a *bool — rather than defaulting them, precisely so this +// is answerable. +// +// It labels records and pointer lines. It decides nothing about re-entry: the +// last poll-served id is taken from what the poll source served, not from +// guessing which lane a delivery came from. +func LaneOf(event eventfeed.Event) Lane { + if event.ActorType == "" && event.VisibleToClients == nil { + return LanePoll + } + return LaneLive +} + +// servedPolls wraps the feed's poll source to record what each page SERVED. +// +// The last poll-served id has to count served events, not delivered ones. The +// package suppresses the poll copy of any event the live lane already +// delivered, and the poll lane runs about thirty seconds behind the live one +// by design, so in steady state nearly every poll page delivers nothing. +// Counting deliveries would leave the id at zero, and every re-entry that +// needs it would fall back to a full replay. This is what the package counts +// for its own reset cursor too. +// +// The repair walk does not go through here: its pages are not the feed's +// position and must never advance the feed's re-entry. +type servedPolls struct { + inner eventfeed.PollSource + + mu sync.Mutex + // byPosition holds the highest id a page served, keyed by the position + // that page issued, until the package confirms the page was delivered. + byPosition map[string]int64 +} + +func (s *servedPolls) Poll(ctx context.Context, cursor eventfeed.Cursor, filters eventfeed.Filters) (eventfeed.PollPage, error) { + page, err := s.inner.Poll(ctx, cursor, filters) + if err != nil || page.Position == "" { + return page, err + } + var highest int64 + for _, event := range page.Events { + highest = max(highest, event.ID) + } + if highest > 0 { + s.mu.Lock() + if s.byPosition == nil { + s.byPosition = make(map[string]int64) + } + s.byPosition[page.Position] = max(s.byPosition[page.Position], highest) + s.mu.Unlock() + } + return page, err +} + +// confirmed returns the highest id served by the page that issued position, +// and forgets every page served before it: pages are polled and delivered one +// at a time, so nothing earlier can still be waiting. +func (s *servedPolls) confirmed(position string) int64 { + s.mu.Lock() + defer s.mu.Unlock() + highest := s.byPosition[position] + clear(s.byPosition) + return highest +} + +// confirmPollServed records the served id of a page the package has just +// finished delivering. Only then: an ingest that failed partway through a page +// leaves the page undelivered and its ids unrecorded. +func (in *Intake) confirmPollServed(ctx context.Context, position string) { + in.mu.Lock() + served := in.served + in.mu.Unlock() + if served == nil { + return + } + highest := served.confirmed(position) + if highest == 0 { + // An empty page. Ordinary — the walk crossed rows the filters exclude + // — and it serves no id, so it advances nothing here. + return + } + if err := in.ledger.NotePollServed(ctx, in.key, highest); err != nil { + in.log.Error("could not record the last poll-served id", "error", err) + return + } + in.mu.Lock() + in.promotedThisRun = true + in.mu.Unlock() +} + +func (in *Intake) observer(ctx context.Context) eventfeed.Observer { + return eventfeed.Observer{ + Connected: func() { in.log.Info("feed socket connected") }, + Confirmed: func() { in.log.Info("feed subscription confirmed") }, + Disconnected: func(reason string, err error) { + // The reason is the peer's own text, and a log is read in a + // terminal like everything else the connector writes. + in.log.Warn("feed socket disconnected", "reason", richtext.SanitizeSingleLine(reason), "error", err) + }, + CatchUpStarted: func(eventfeed.Cursor) { in.log.Info("feed catch-up walk started") }, + PageDelivered: func(_ int, position string) { + // Detached deliberately: the position the page just moved must be + // recorded even if the run's context is on its way down, or a + // shutdown mid-page loses the id the next re-entry needs. + in.confirmPollServed(context.WithoutCancel(ctx), position) //nolint:contextcheck // detached on purpose, see above + }, + CaughtUp: func() { + // A replay that reached the head without meeting the epoch is + // over; nothing after this is part of it. + in.setReplaying(false) + // "Caught up with the walk", not "caught up with the account": + // delivery has write-time brakes that write no addressing and say + // nothing, so a quiet feed is never proof of a quiet project. + in.log.Info("feed walk reached its head and the buffer drained") + }, + Checkpoint: func(string) { + in.mu.Lock() + in.checkpointed = true + // A saved position is progress, whether or not its page carried + // an event: this connection is no longer a re-entry waiting to + // prove itself, and a later refusal is an ordinary one. + in.enteredByReentry = false + in.mu.Unlock() + }, + CheckpointSaveFailed: func(err error) { + in.log.Error("could not save the feed position", "error", err) + }, + Gap: func(epochAfterID int64, resumeOrigin string) { + in.log.Warn("feed served a 410", "epoch_after_id", epochAfterID, "resume_origin", resumeOrigin) + }, + PositionRejected: func(kind eventfeed.PollErrorKind) { + in.log.Warn("feed rejected the held position", "kind", kind.String()) + in.onPositionRejected(ctx) + }, + FilterConflict: func(positionDigest, filtersDigest string) { + in.log.Warn("feed position was minted for a different filter set", + "position_digest", positionDigest, "filters_digest", filtersDigest) + }, + StaleConnection: func(d time.Duration) { + in.log.Warn("feed socket went stale", "since_last_frame", d) + }, + BufferOverflow: func(dropped int) { + in.log.Warn("live buffer overflowed", "dropped", dropped) + }, + } +} + +// handleSignal decides what a semantic signal means for this connector. It +// runs synchronously on the delivery path, so it does only what must happen +// before the disposition takes effect and starts the rest elsewhere. +func (in *Intake) handleSignal(signal eventfeed.Signal) eventfeed.Disposition { + ctx := context.WithoutCancel(context.Background()) + + switch s := signal.(type) { + case eventfeed.FeedGap: + // The ACCOUNT lane's 410, and the only one that reaches here: the + // adapter never maps a retention 410 onto this signal. The resume URL + // is followed exactly as served — the entry class is the server's + // decision, read out of its cursor, never substituted. + epoch := s.EpochAfterID + note := "the feed's served history before the epoch is gone" + in.mu.Lock() + if in.replaying { + // Expected, not a loss: this connection chose to replay from the + // beginning to recover, and the beginning is below the epoch. The + // label covers that one entry; any later 410 on this connection is + // a real gap and is recorded as one. + note = "a recovery replay from the beginning of served history met the epoch, as expected; not a loss" + in.replaying = false + } + in.mu.Unlock() + if _, err := in.ledger.RecordGap(ctx, Gap{ + DetectedAt: in.now(), + Class: GapEpoch, + EpochAfterID: &epoch, + EntryClass: entryClassOf(s.ResumeURL), + Note: note, + }); err != nil { + // A gap we cannot write down is a gap nothing will ever report. + in.log.Error("could not record the feed gap; refusing to continue past it", "error", err) + return eventfeed.Terminate + } + return eventfeed.Accept + + case eventfeed.BufferOverflow: + loss, err := in.ledger.RecordLoss(ctx, s.DroppedIDs, in.now(), in.opts.RepairWindow, in.opts.Filters) + if err != nil { + // Accept means owning the incompleteness. Owning it begins with + // it being on disk: accepting after a failed write would leave a + // loss that no restart could ever find, which is the one outcome + // worse than terminating. + in.log.Error("could not record the buffer overflow; refusing to accept it", "error", err) + return eventfeed.Terminate + } + in.log.Warn("live buffer overflowed; reconciling", + "dropped", s.DroppedCount, "loss_id", loss.ID, "repair_since", loss.RepairSince) + in.startRepair(loss) + return eventfeed.Accept + } + return eventfeed.Terminate +} + +// entryClassOf reads the entry class out of the cursor the server served. +func entryClassOf(resumeURL string) EntryClass { + parsed, err := url.Parse(resumeURL) + if err != nil { + return EntryUnknown + } + switch since := parsed.Query().Get("since"); since { + case "now": + return EntryPresent + case "": + return EntryUnknown + default: + if _, err := strconv.ParseInt(since, 10, 64); err != nil { + return EntryUnknown + } + return EntryReplay + } +} + +// strandedIDs is the ids a handover failed on, oldest first, as the feed +// served them. +func (in *Intake) strandedIDs() []int64 { + in.mu.Lock() + defer in.mu.Unlock() + ids := make([]int64, 0, len(in.stranded)) + for id := range in.stranded { + ids = append(ids, id) + } + slices.Sort(ids) + return ids +} + +// strand remembers an id the ledger has but the queue does not. +// +// Every commit goes through ingest, and every handover failure after a commit +// goes through here, so there is one place an event can be stranded and one +// place that answers for it. A crash loses the list, and nothing is lost with +// it: the next start offers every record still in seen. +func (in *Intake) strand(id int64) { + in.mu.Lock() + defer in.mu.Unlock() + if in.stranded == nil { + in.stranded = map[int64]struct{}{} + } + in.stranded[id] = struct{}{} +} + +// sweepStranded offers the ids a failed handover left behind. +// +// A restart is not an answer for a connector that runs for weeks, and a +// repair walk is the case that makes it urgent: its failure closes nothing +// downstream, the id's loss row is already resolved by the commit, and the +// reconciliation that follows sees nothing missing. The event would be +// recorded, never judged, and never mentioned again. +// +// It offers and only then forgets: an offer refused by a canceled context +// leaves the id stranded for the next sweep, or for the next start. +func (in *Intake) sweepStranded(ctx context.Context) { + ids := in.strandedIDs() + if len(ids) == 0 { + return + } + for _, id := range ids { + if err := in.queue.Offer(ctx, id); err != nil { + in.log.Warn("an event the ledger holds could not be handed over; it stays for the next sweep", "event_id", id, "error", err) + return + } + in.mu.Lock() + delete(in.stranded, id) + in.mu.Unlock() + } +} + +// requeueSeen hands every record still in seen to the queue. +// +// The ledger row is written before the pointer line and before the hand-off, +// so that a crash can never lose the pointer — but that ordering means a +// failure after the commit leaves an event the ledger's own dedupe will +// suppress on every retry. Nothing may be left in that state, so a start +// offers every record nothing has judged yet. Offering one twice costs +// nothing: the queue carries ids, and admission moves a record out of seen +// before it acts. +// +// The pointer line is NOT written again here. A record that got no line when +// it was recorded — the write that failed — never gets one, and the work still +// happens through the queue. Re-emitting would put duplicate lines in front of +// every watcher for the ordinary case, to close a gap in the stream rather +// than in the work. +func (in *Intake) requeueSeen(ctx context.Context) error { + // What is cleared at the end is what was stranded when this began. A + // repair walk is already running by now, and it serves OLD ids, which the + // paging below may already have passed: one stranded mid-pass would be + // forgotten by a clear that assumed it had offered everything. + carried := in.strandedIDs() + var after int64 + for { + records, err := in.ledger.RecordsInStateAfter(ctx, StateSeen, after, requeueBatch) + if err != nil { + return err + } + for _, record := range records { + if err := in.queue.Offer(ctx, record.ID); err != nil { + // A shutdown mid-requeue leaves the rest recorded and still + // seen, which the next start offers again. + return err + } + after = record.ID + } + if len(records) < requeueBatch { + // Everything in seen has just been offered, the ids a previous + // run stranded included. + in.mu.Lock() + for _, id := range carried { + delete(in.stranded, id) + } + in.mu.Unlock() + return nil + } + } +} + +// requeueBatch is how many seen records a start reads at a time. +const requeueBatch = 500 + +// resumeReconciliation restarts every open loss's repair walk on start. A +// crash between the overflow and its repair is a delay, not a loss of the +// record. +func (in *Intake) resumeReconciliation(ctx context.Context) error { + in.startRepairWorkers(ctx) + losses, err := in.ledger.OpenLosses(ctx) + if err != nil { + return err + } + for _, loss := range losses { + in.log.Info("resuming reconciliation of an open loss", "loss_id", loss.ID, "repair_since", loss.RepairSince) + in.startRepair(loss) + } + return nil +} + +// startRepair hands a loss to the repair workers. +// +// Repairs are bounded: an overloaded feed can raise an overflow per dropped +// event, and a goroutine and a poll source per loss would answer an API that +// is already struggling with a storm of walks. A loss that finds the queue +// full is left open on disk, which the next start resumes. +func (in *Intake) startRepair(loss Loss) { + if loss.ResolvedAt != nil { + return + } + in.mu.Lock() + queue := in.repairQueue + in.mu.Unlock() + if queue == nil { + // No pool: nobody would walk this loss. It stays open on disk, and + // the next start resumes it. + in.log.Warn("no repair workers are running; this loss stays open for the next start", "loss_id", loss.ID) + return + } + in.mu.Lock() + if in.inFlight == nil { + in.inFlight = make(map[int64]bool) + } + if in.inFlight[loss.ID] { + in.mu.Unlock() + return + } + in.inFlight[loss.ID] = true + in.mu.Unlock() + + select { + case queue <- loss: + default: + // No room now. The sweeper offers it again, so a loss is never left + // with nothing that will attempt it. + in.mu.Lock() + delete(in.inFlight, loss.ID) + in.mu.Unlock() + in.log.Warn("the repair queue is full; this loss is left for the sweep", "loss_id", loss.ID) + } +} + +// startRepairWorkers starts the fixed repair pool, once. +// +// Every Add happens here, on the goroutine that later waits, before anything +// can wait: a worker that added itself as it picked work up would be adding to +// a group a shutdown may already be waiting on, which Go refuses outright. +func (in *Intake) startRepairWorkers(ctx context.Context) { + in.mu.Lock() + if in.repairQueue != nil { + in.mu.Unlock() + return + } + size := in.repairQueueSize + if size <= 0 { + size = repairQueueDepth + } + queue := make(chan Loss, size) + in.repairQueue = queue + in.mu.Unlock() + for range maxConcurrentRepairs { + in.repairs.Add(1) + go in.repairWorker(ctx, queue) + } + in.repairs.Add(1) + go in.sweepLosses(ctx) +} + +// sweepLosses is the periodic repair of both things that can be left behind: +// an event committed but never handed over, and an open loss nothing is +// walking. +// +// One goroutine does both, so a handover that waits at the pause threshold +// also holds up the re-offering of open losses. That is the right way round: +// the backlog is full, the pipeline is stopped, and starting more repair +// walks would only make the backlog worse. +// +// The queue is bounded, so an overloaded connector can turn one away; and a +// walk can end early, leaving its loss open. Neither may leave a loss with +// nothing that will attempt it again, and waiting for a restart is not an +// answer for a connector that runs for weeks. The sweep is what closes that: +// every open loss is either being walked, or is offered again here. +func (in *Intake) sweepLosses(ctx context.Context) { + defer in.repairs.Done() + every := in.repairSweep + if every <= 0 { + every = defaultRepairSweep + } + ticker := time.NewTicker(every) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + in.sweepStranded(ctx) + losses, err := in.ledger.OpenLosses(ctx) + if err != nil { + in.log.Warn("could not read the open losses", "error", err) + continue + } + for _, loss := range losses { + in.startRepair(loss) + } + } + } +} + +// defaultRepairSweep is how often the open losses are re-offered. +const defaultRepairSweep = time.Minute + +// releaseRepairWorkers forgets a pool whose workers have stopped, so the next +// Run starts its own. A queue left in place would take losses nobody walks, +// and say nothing. +func (in *Intake) releaseRepairWorkers() { + in.mu.Lock() + defer in.mu.Unlock() + in.repairQueue = nil + // The marks say "a worker has this", and the workers are gone. A loss + // still sitting in the dropped queue would otherwise be skipped by every + // later run and every sweep. + in.inFlight = nil +} + +// repairWorker walks one loss at a time until ctx ends. +// +// A walk holds its worker for as long as it takes, the waits between its +// passes included, so a loss can sit in the queue past its own window and get +// the single catch-up pass a closed window allows. Recovery still happens; it +// happens later. +func (in *Intake) repairWorker(ctx context.Context, queue chan Loss) { + defer in.repairs.Done() + for { + select { + case <-ctx.Done(): + return + case loss := <-queue: + in.runRepair(ctx, loss) + } + } +} + +func (in *Intake) runRepair(ctx context.Context, loss Loss) { + defer func() { + in.mu.Lock() + delete(in.inFlight, loss.ID) + in.mu.Unlock() + }() + walker := &repairWalker{ + ledger: in.ledger, + polls: in.opts.PollsFor(), + origin: in.key.Origin, + filters: in.opts.Filters, + ingest: in.ingest, + now: in.now, + interval: in.opts.RepairInterval, + log: in.log, + sleep: in.repairSleep, + } + switch err := walker.reconcileLoss(ctx, loss); { + case err == nil: + case errors.Is(err, context.Canceled), errors.Is(err, context.DeadlineExceeded): + // A shutdown mid-walk is a delay: the loss is still open on disk + // and the next start picks it up where this one left off. + in.log.Info("reconciliation paused by shutdown; it resumes on the next start", "loss_id", loss.ID) + default: + in.log.Error("reconciliation of a loss ended early", "loss_id", loss.ID, "error", err) + } +} + +// takeSnapshot records the buckets the lister names at the moment this +// connection subscribes — the same set the cable snapshots. +func (in *Intake) takeSnapshot(ctx context.Context) { + if in.opts.Membership == nil { + return + } + buckets, err := in.opts.Membership.Buckets(ctx) + if err != nil { + in.log.Warn("could not read the agent's projects; keeping the previous listing", "error", err) + return + } + in.adoptListing(buckets) +} + +// adoptListing replaces the authoritative set and revokes every learned bucket +// the listing does not name. It reports whether the listing changed; the first +// listing is a baseline, not a change. +func (in *Intake) adoptListing(buckets []int64) bool { + in.mu.Lock() + defer in.mu.Unlock() + listed := make(map[int64]bool, len(buckets)) + for _, id := range buckets { + listed[id] = true + } + // A learned bucket is provisional, and the lister is the trust boundary: + // one it no longer names is revoked, however recently it was learned, and + // the next event from it is unknown again. + for id := range in.learned { + if !listed[id] { + delete(in.learned, id) + in.log.Info("a project the lister no longer names is no longer held", "bucket_id", id) + } + } + changed := in.listed != nil && !sameBuckets(in.listed, listed) + in.listed = listed + return changed +} + +func sameBuckets(a, b map[int64]bool) bool { + if len(a) != len(b) { + return false + } + for id := range a { + if !b[id] { + return false + } + } + return true +} + +// noteBucket asks for a reconnect when an event arrives from a bucket the live +// subscription may not hold. The poll lane authorizes at read time, so it +// covers the project immediately; the live lane cannot until it re-subscribes. +func (in *Intake) noteBucket(bucketID int64) { + in.mu.Lock() + known := in.opts.Membership == nil || in.listed[bucketID] || in.learned[bucketID] + if !known { + // A bucket revoked by a listing is unknown again, but relearning it + // is rate-limited: without that, a project the lister never names — + // archived, or past its page — would cost a reconnect per event. + if since, ok := in.relearned[bucketID]; ok && in.now().Sub(since) < in.opts.MembershipInterval { + known = true + } else { + if in.learned == nil { + in.learned = make(map[int64]bool) + in.relearned = make(map[int64]time.Time) + } + in.learned[bucketID] = true + in.relearned[bucketID] = in.now() + } + } + in.mu.Unlock() + if known { + return + } + in.log.Info("an event arrived from a project the live subscription does not hold", "bucket_id", bucketID) + in.requestReconnect() +} + +// onPositionRejected handles a refused position on a connection that has not +// yet recorded a poll-served id of its own. +// +// The package re-enters from an in-memory id that starts at zero each run, so +// right after a restart it would take the present and skip everything since +// the refused position. The connection is ended before that re-entry polls, +// and remade at a safe entry: +// +// - this filter set's own last poll-served id, when it has one; +// - otherwise the beginning of served history. A position can be saved +// from empty pages alone, so "no poll-served id" does not mean "nothing +// to skip". Replaying costs reads the ledger's dedupe absorbs; entering at +// the present, or at another filter set's id, costs events. +func (in *Intake) onPositionRejected(ctx context.Context) { + in.mu.Lock() + promoted := in.promotedThisRun + reentered := in.enteredByReentry + in.mu.Unlock() + if !promoted && reentered { + // The safe re-entry was itself refused before it served a page. + // Whatever the server objects to, another re-entry will not cure it, + // and reconnecting again would mint, dial and poll in a tight loop. + in.abort(errors.New("connector: the feed refused the safe re-entry after a refused position; not retrying")) + return + } + if promoted { + // The package's own reset cursor is this run's poll-served id, which + // is at least what the ledger holds. + return + } + served, err := in.positions.LastPollServedID(ctx, in.key) + if err != nil { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + // The connection is already ending — a reconnect or a shutdown — + // so the package's re-entry will not poll. Nothing to decide. + return + } + // Not knowing where it is safe to re-enter is a reason to stop, not + // to guess. + in.abort(fmt.Errorf("connector: a position was refused and the safe re-entry could not be read: %w", err)) + return + } + + in.mu.Lock() + if served > 0 { + in.reentry = eventfeed.StartAfter(served) + in.reentryLog = "re-entering after this filter set's last poll-served id " + strconv.FormatInt(served, 10) + } else { + in.reentry = eventfeed.StartBeginning() + in.reentryLog = "no poll-served id for this filter set; re-entering at the beginning of served history" + } + in.reentryReplays = served == 0 + in.hasReentry = true + in.mu.Unlock() + in.requestReconnect() +} + +func (in *Intake) setReplaying(replaying bool) { + in.mu.Lock() + in.replaying = replaying + in.mu.Unlock() +} + +func (in *Intake) checkpointSaved() bool { + in.mu.Lock() + defer in.mu.Unlock() + return in.checkpointed +} + +// abort ends the current connection and the run with err. +func (in *Intake) abort(err error) { + in.mu.Lock() + if in.abortErr == nil { + in.abortErr = err + } + cancel := in.cancelRun + in.mu.Unlock() + if cancel != nil { + cancel() + } +} + +// watchMembership re-reads the agent's projects on a timer and asks for a +// reconnect when the set changes. It runs until ctx ends; the returned +// function waits for it to have stopped. +func (in *Intake) watchMembership(ctx context.Context) func() { + if in.opts.Membership == nil { + return func() {} + } + done := make(chan struct{}) + go func() { + defer close(done) + retry := in.membershipRetry + if retry <= 0 { + retry = defaultMembershipRetry + } + for { + // A snapshot the subscribe-time read never produced is retried on + // a backoff rather than left for a full interval: until one exists + // there is no baseline to notice a change against. + wait := in.opts.MembershipInterval + if !in.hasSnapshot() { + wait = min(retry, in.opts.MembershipInterval) + retry = min(retry*2, in.opts.MembershipInterval) + } + timer := time.NewTimer(wait) + select { + case <-ctx.Done(): + timer.Stop() + return + case <-timer.C: + } + buckets, err := in.opts.Membership.Buckets(ctx) + if err != nil { + // A failed read leaves the snapshot alone. Treating it as a + // change would reconnect the feed every time the API + // hiccupped. + in.log.Warn("could not refresh the agent's projects", "error", err) + continue + } + if in.membershipChanged(buckets) { + in.requestReconnect() + return + } + } + }() + return func() { <-done } +} + +// maxConcurrentRepairs bounds repair walks in flight, and repairQueueDepth +// the losses waiting for one. +const ( + maxConcurrentRepairs = 2 + repairQueueDepth = 1024 +) + +// defaultMembershipRetry is the first wait before re-reading a membership +// listing that failed; it doubles up to the membership interval. +const defaultMembershipRetry = 5 * time.Second + +func (in *Intake) hasSnapshot() bool { + in.mu.Lock() + defer in.mu.Unlock() + return in.listed != nil +} + +// membershipChanged adopts a fresh listing and reports whether the +// authoritative set changed. +func (in *Intake) membershipChanged(buckets []int64) bool { + return in.adoptListing(buckets) +} + +// requestReconnect marks a reconnect due and ends the current connection now, +// rather than whenever the feed next yields. +func (in *Intake) requestReconnect() { + select { + case in.reconnect <- struct{}{}: + default: + } + in.mu.Lock() + cancel := in.cancelRun + in.mu.Unlock() + if cancel != nil { + cancel() + } +} + +func (in *Intake) reconnectRequested() bool { + select { + case <-in.reconnect: + return true + default: + return false + } +} + +// pointerWriter writes the NDJSON pointer lines through the connector's shared +// line writer: whole lines, never interleaved. +type pointerWriter struct { + out *ndjson.Writer +} + +func newPointerWriter(w io.Writer, shared *ndjson.Writer) *pointerWriter { + if shared != nil { + return &pointerWriter{out: shared} + } + return &pointerWriter{out: ndjson.NewWriter(w)} +} + +// Pointer is the line intake writes to stdout for each newly seen event. It +// carries what the feed carried and nothing more: no title, no body, no URL, +// no names. Whoever wants those pays for a read. +type Pointer struct { + EventID int64 `json:"event_id"` + EventType string `json:"event_type"` + Kind string `json:"kind"` + Action string `json:"action"` + BucketID int64 `json:"bucket_id"` + CreatorID int64 `json:"creator_id"` + PerformedByID *int64 `json:"performed_by_id"` + RecordingID int64 `json:"recording_id"` + CreatedAt string `json:"created_at"` + Lane Lane `json:"lane"` + State string `json:"state"` +} + +func (p *pointerWriter) write(event eventfeed.Event, lane Lane) error { + // Every string from Basecamp is stripped of terminal controls. The line is + // a wire, but it is also what a person watching the connector sees: JSON + // escapes C0 controls but passes C1 controls such as U+009B (CSI) through + // as raw UTF-8, which a terminal executes. Admission's lines apply the + // same rule. + clean := richtext.SanitizeTerminal + err := p.out.WriteLine(Pointer{ + EventID: event.ID, + EventType: clean(event.EventType), + Kind: clean(event.Kind), + Action: clean(event.Action), + BucketID: event.BucketID, + CreatorID: event.CreatorID, + PerformedByID: event.PerformedByID, + RecordingID: event.RecordingID, + CreatedAt: event.CreatedAt.UTC().Format(time.RFC3339Nano), + Lane: lane, + State: string(StateSeen), + }) + if err != nil { + return fmt.Errorf("connector: pointer line: %w", err) + } + return nil +} diff --git a/internal/connector/intake_feed_test.go b/internal/connector/intake_feed_test.go new file mode 100644 index 000000000..d3b7dfc25 --- /dev/null +++ b/internal/connector/intake_feed_test.go @@ -0,0 +1,381 @@ +package connector + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed/feedtest" +) + +// End-to-end through the real eventfeed run loop, on the package's own +// deterministic fakes: subscribe, catch up, drain, stream — and then a restart +// over the same ledger. + +type safeBuffer struct { + mu sync.Mutex + buf bytes.Buffer +} + +func (b *safeBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.Write(p) +} + +func (b *safeBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.String() +} + +func pushFrame(t *testing.T, identifier string, event eventfeed.Event) []byte { + t.Helper() + payload := map[string]any{ + "id": event.ID, + "kind": event.Kind, + "event_type": event.EventType, + "action": event.Action, + "created_at": event.CreatedAt.UTC().Format(time.RFC3339), + "bucket_id": event.BucketID, + "creator_id": event.CreatorID, + "performed_by_id": nil, + "actor_type": "person", + "recording_id": event.RecordingID, + "visible_to_clients": true, + } + raw, err := json.Marshal(payload) + require.NoError(t, err) + frame, err := json.Marshal(map[string]json.RawMessage{ + "identifier": mustJSON(t, identifier), + "message": raw, + }) + require.NoError(t, err) + return frame +} + +func confirmFrame(t *testing.T, identifier string) []byte { + t.Helper() + frame, err := json.Marshal(map[string]any{ + "type": "confirm_subscription", + "identifier": identifier, + }) + require.NoError(t, err) + return frame +} + +func mustJSON(t *testing.T, v any) json.RawMessage { + t.Helper() + raw, err := json.Marshal(v) + require.NoError(t, err) + return raw +} + +// subscribedConn waits for the connector's subscribe command and answers it, +// returning the connection and the identifier the connector chose. +func subscribedConn(t *testing.T, transport *feedtest.Transport) (*feedtest.Conn, string) { + t.Helper() + var ( + conn *feedtest.Conn + identifier string + ) + require.Eventually(t, func() bool { + conn = transport.LastConn() + return conn != nil + }, 5*time.Second, 5*time.Millisecond, "the connector should dial the cable URL the mint served") + + // Action Cable greets first; the subscribe follows the welcome. + conn.Serve([]byte(`{"type":"welcome"}`)) + + require.Eventually(t, func() bool { + writes := conn.Writes() + if len(writes) == 0 { + return false + } + var command struct { + Command string `json:"command"` + Identifier string `json:"identifier"` + } + if err := json.Unmarshal(writes[0], &command); err != nil { + return false + } + identifier = command.Identifier + return command.Command == "subscribe" && identifier != "" + }, 5*time.Second, 5*time.Millisecond, "the connector should subscribe before it takes a position") + + conn.Serve(confirmFrame(t, identifier)) + return conn, identifier +} + +func TestIntakeRunsTheFeedThroughCatchUpAndStreaming(t *testing.T) { + ledger := newTestLedger(t) + queue, err := NewQueue(DefaultBacklogWarn, DefaultBacklogPause) + require.NoError(t, err) + + transport := feedtest.NewTransport() + minter := feedtest.NewMinter() + minter.ScriptTicket(eventfeed.StreamTicket{Ticket: "t", ExpiresIn: 120, URL: "wss://cable.basecamp.com/cable?ticket=t"}) + polls := feedtest.NewPolls() + polls.ScriptPage(eventfeed.PollPage{ + Events: []eventfeed.Event{testEvent(17099838500)}, + Position: "caught-up-position", + }) + + var pointers safeBuffer + intake, err := New(Options{ + Origin: "https://3.basecampapi.com", + AccountID: "2914079", + ConsumerNamespace: "connector-test", + Ledger: ledger, + Queue: queue, + Minter: minter, + Polls: polls, + Transport: transport, + Pointers: &pointers, + }) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan error, 1) + go func() { done <- intake.Run(ctx) }() + + conn, identifier := subscribedConn(t, transport) + + // The catch-up page lands first; then a live event on the socket. + require.Eventually(t, func() bool { + _, ok, err := ledger.Get(ctx, 17099838500) + return err == nil && ok + }, 5*time.Second, 5*time.Millisecond, "the catch-up walk's events reach the ledger") + + conn.Serve(pushFrame(t, identifier, testEvent(17099838600))) + require.Eventually(t, func() bool { + _, ok, err := ledger.Get(ctx, 17099838600) + return err == nil && ok + }, 5*time.Second, 5*time.Millisecond, "a live event reaches the ledger") + + // Only the poll lane advances the durable position. + position, ok, err := ledger.Load(ctx, intake.CheckpointKey()) + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, "caught-up-position", position) + + live, _, err := ledger.Get(ctx, 17099838600) + require.NoError(t, err) + assert.Equal(t, LaneLive, live.Lane) + polled, _, err := ledger.Get(ctx, 17099838500) + require.NoError(t, err) + assert.Equal(t, LanePoll, polled.Lane) + + assert.Equal(t, 2, countLines(pointers.String())) + assert.Equal(t, 2, queue.Depth()) + + cancel() + select { + case err := <-done: + assert.ErrorIs(t, err, context.Canceled) + case <-time.After(5 * time.Second): + t.Fatal("Run should return when its context is canceled") + } +} + +// Kill and restart: no duplicate, and the walk resumes from the stored +// position rather than the present. +func TestIntakeSurvivesARestartWithoutDuplicating(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "state", "connector.db") + + firstLedger, err := OpenLedger(path) + require.NoError(t, err) + _, err = firstLedger.RecordSeen(context.Background(), testEvent(17099838500), LanePoll) + require.NoError(t, err) + require.NoError(t, firstLedger.Save(context.Background(), eventfeed.CheckpointKey{ + Origin: "https://3.basecampapi.com", + AccountID: "2914079", + ConsumerNamespace: "connector-test", + FilterKey: eventfeed.Filters{}.FilterKey(), + }, "position-from-the-previous-run")) + require.NoError(t, firstLedger.Close()) + + ledger, err := OpenLedger(path) + require.NoError(t, err) + t.Cleanup(func() { _ = ledger.Close() }) + + queue, err := NewQueue(DefaultBacklogWarn, DefaultBacklogPause) + require.NoError(t, err) + + transport := feedtest.NewTransport() + minter := feedtest.NewMinter() + minter.ScriptTicket(eventfeed.StreamTicket{Ticket: "t", ExpiresIn: 120, URL: "wss://cable.basecamp.com/cable?ticket=t"}) + polls := feedtest.NewPolls() + // The same event the previous run already saw, plus one it did not. + polls.ScriptPage(eventfeed.PollPage{ + Events: []eventfeed.Event{testEvent(17099838500), testEvent(17099838501)}, + Position: "position-after-the-restart", + }) + + var pointers safeBuffer + intake, err := New(Options{ + Origin: "https://3.basecampapi.com", + AccountID: "2914079", + ConsumerNamespace: "connector-test", + Ledger: ledger, + Queue: queue, + Minter: minter, + Polls: polls, + Transport: transport, + Pointers: &pointers, + }) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan error, 1) + go func() { done <- intake.Run(ctx) }() + + subscribedConn(t, transport) + + require.Eventually(t, func() bool { + _, ok, err := ledger.Get(ctx, 17099838501) + return err == nil && ok + }, 5*time.Second, 5*time.Millisecond) + + calls := polls.Calls() + require.NotEmpty(t, calls) + assert.Equal(t, "position-from-the-previous-run", calls[0].Cursor.Position, + "a restart resumes from the stored position, never at the head") + + // The ledger row is written before the pointer line and the hand-off, so + // the counts are asserted on their own terms. + // + // One pointer line: the event the previous run already saw is not written + // out again. Two ids in the queue: that event was left unjudged in the + // ledger, so this start hands it over too, exactly once. + require.Eventually(t, func() bool { return countLines(pointers.String()) == 1 && queue.Depth() == 2 }, + 5*time.Second, 5*time.Millisecond, + "the event the previous run already saw is not written out again, and the unjudged one is handed over") + assert.Equal(t, 1, countLines(pointers.String())) + queued := map[int64]int{} + for range 2 { + id, err := queue.Take(ctx) + require.NoError(t, err) + queued[id]++ + } + assert.Equal(t, map[int64]int{17099838500: 1, 17099838501: 1}, queued) + + cancel() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("Run should return when its context is canceled") + } +} + +func TestIntakeRecordsTheLastPollServedIDFromTheWalk(t *testing.T) { + ledger := newTestLedger(t) + queue, err := NewQueue(DefaultBacklogWarn, DefaultBacklogPause) + require.NoError(t, err) + + transport := feedtest.NewTransport() + minter := feedtest.NewMinter() + minter.ScriptTicket(eventfeed.StreamTicket{Ticket: "t", ExpiresIn: 120, URL: "wss://cable.basecamp.com/cable?ticket=t"}) + polls := feedtest.NewPolls() + polls.ScriptPage(eventfeed.PollPage{ + Events: []eventfeed.Event{testEvent(17099838500), testEvent(17099838501)}, + Position: "p1", + }) + + intake, err := New(Options{ + Origin: "https://3.basecampapi.com", + AccountID: "2914079", + ConsumerNamespace: "connector-test", + Ledger: ledger, + Queue: queue, + Minter: minter, + Polls: polls, + Transport: transport, + }) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go func() { _ = intake.Run(ctx) }() + + conn, identifier := subscribedConn(t, transport) + + require.Eventually(t, func() bool { + served, err := ledger.LastPollServedID(ctx, intake.CheckpointKey()) + return err == nil && served == 17099838501 + }, 5*time.Second, 5*time.Millisecond, "the poll lane's highest served id is durable") + + // A live event far ahead of the poll lane must not move it. + conn.Serve(pushFrame(t, identifier, testEvent(17099999999))) + require.Eventually(t, func() bool { + _, ok, err := ledger.Get(ctx, 17099999999) + return err == nil && ok + }, 5*time.Second, 5*time.Millisecond) + + served, err := ledger.LastPollServedID(ctx, intake.CheckpointKey()) + require.NoError(t, err) + assert.Equal(t, int64(17099838501), served, + fmt.Sprintf("a live id is not a poll-served id; re-entering at %d would skip the safety delay", 17099999999)) +} + +// A repair walk must not strand the process on the way out. It is bound to +// Run's lifetime, and an unfinished walk is a delay: the loss is still open on +// disk and the next start resumes it. +func TestShutdownDoesNotWaitOutAnOpenRepairWalk(t *testing.T) { + ledger := newTestLedger(t) + queue, err := NewQueue(DefaultBacklogWarn, DefaultBacklogPause) + require.NoError(t, err) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + _, err = ledger.RecordLoss(ctx, []int64{17099838509}, time.Now(), time.Hour, eventfeed.Filters{}) + require.NoError(t, err) + + transport := feedtest.NewTransport() + minter := feedtest.NewMinter() + minter.ScriptTicket(eventfeed.StreamTicket{Ticket: "t", ExpiresIn: 120, URL: "wss://cable.basecamp.com/cable?ticket=t"}) + polls := feedtest.NewPolls() + + intake, err := New(Options{ + Origin: "https://3.basecampapi.com", + AccountID: "2914079", + ConsumerNamespace: "connector-test", + Ledger: ledger, + Queue: queue, + Minter: minter, + Polls: polls, + Transport: transport, + // A repair cadence far longer than the test: the walk is asleep when + // the shutdown arrives, which is the case that used to hang. + RepairInterval: time.Hour, + }) + require.NoError(t, err) + + done := make(chan error, 1) + go func() { done <- intake.Run(ctx) }() + + subscribedConn(t, transport) + cancel() + + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("Run should return on shutdown rather than wait out the repair cadence") + } + + open, err := ledger.OpenLosses(context.Background()) + require.NoError(t, err) + assert.Len(t, open, 1, "the loss stays on disk for the next start to resume") +} diff --git a/internal/connector/intake_test.go b/internal/connector/intake_test.go new file mode 100644 index 000000000..3ff36c074 --- /dev/null +++ b/internal/connector/intake_test.go @@ -0,0 +1,387 @@ +package connector + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "io" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" + + "github.com/basecamp/basecamp-cli/internal/connector/ndjson" +) + +type stubMinter struct{} + +func (stubMinter) MintStreamTicket(context.Context) (eventfeed.StreamTicket, error) { + return eventfeed.StreamTicket{}, nil +} + +type scriptedPolls struct { + pages []eventfeed.PollPage + errs []error + cursors []eventfeed.Cursor + filters []eventfeed.Filters + calls int +} + +func (s *scriptedPolls) Poll(_ context.Context, cursor eventfeed.Cursor, filters eventfeed.Filters) (eventfeed.PollPage, error) { + s.cursors = append(s.cursors, cursor) + s.filters = append(s.filters, filters) + i := s.calls + s.calls++ + if i < len(s.errs) && s.errs[i] != nil { + return eventfeed.PollPage{}, s.errs[i] + } + if i < len(s.pages) { + return s.pages[i], nil + } + return eventfeed.PollPage{Position: "drained"}, nil +} + +type fixedClock struct{ at time.Time } + +func (c *fixedClock) now() time.Time { return c.at } + +func newTestIntake(t *testing.T, polls eventfeed.PollSource, pointers io.Writer) (*Intake, *Ledger, *Queue) { + t.Helper() + return newTestIntakeWith(t, newTestLedger(t), polls, pointers) +} + +// newTestIntakeLines builds an intake whose pointer lines go through an +// injected writer. +func newTestIntakeLines(t *testing.T, lines *ndjson.Writer) (*Intake, *Ledger, *Queue) { + t.Helper() + intake, ledger, queue := newTestIntakeWith(t, newTestLedger(t), nil, nil) + intake.pointer = newPointerWriter(linesSink(lines), lines) + return intake, ledger, queue +} + +// newTestIntakeOn builds an intake over an existing ledger, as a later start +// would. +func newTestIntakeOn(t *testing.T, ledger *Ledger, pointers io.Writer) (*Intake, *Ledger, *Queue) { + t.Helper() + return newTestIntakeWith(t, ledger, nil, pointers) +} + +func newTestIntakeWith(t *testing.T, ledger *Ledger, polls eventfeed.PollSource, pointers io.Writer) (*Intake, *Ledger, *Queue) { + t.Helper() + queue, err := NewQueue(DefaultBacklogWarn, DefaultBacklogPause) + require.NoError(t, err) + if polls == nil { + polls = &scriptedPolls{} + } + intake, err := New(Options{ + Origin: "https://3.basecampapi.com", + AccountID: "2914079", + ConsumerNamespace: "connector-test", + Ledger: ledger, + Queue: queue, + Minter: stubMinter{}, + Polls: polls, + Pointers: pointers, + Clock: (&fixedClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)}).now, + }) + require.NoError(t, err) + return intake, ledger, queue +} + +func TestLaneOfReadsThePushLaneTransportFields(t *testing.T) { + poll := testEvent(1) + assert.Equal(t, LanePoll, LaneOf(poll), "poll rows omit actor_type and visible_to_clients") + + visible := true + push := testEvent(2) + push.ActorType = "person" + push.VisibleToClients = &visible + assert.Equal(t, LaneLive, LaneOf(push)) + + // Presence, not value: visible_to_clients false is still present. + hidden := false + pushHidden := testEvent(3) + pushHidden.ActorType = "agent" + pushHidden.VisibleToClients = &hidden + assert.Equal(t, LaneLive, LaneOf(pushHidden)) +} + +func TestIngestWritesOnePointerAndQueuesOnce(t *testing.T) { + var pointers bytes.Buffer + intake, _, queue := newTestIntake(t, nil, &pointers) + ctx := context.Background() + + require.NoError(t, intake.ingest(ctx, testEvent(17099838500), LaneLive)) + require.NoError(t, intake.ingest(ctx, testEvent(17099838500), LanePoll)) + + assert.Equal(t, 1, queue.Depth(), "the poll lane repeating the live lane is not a second unit of work") + assert.Equal(t, 1, countLines(pointers.String())) + + var pointer Pointer + require.NoError(t, json.Unmarshal([]byte(strings.TrimSpace(pointers.String())), &pointer)) + assert.Equal(t, int64(17099838500), pointer.EventID) + assert.Equal(t, "comment.created", pointer.EventType) + assert.Equal(t, int64(10304028972), pointer.RecordingID) + assert.Equal(t, string(StateSeen), pointer.State) +} + +// The pointer carries what the feed carried and nothing more. Anything with a +// title or a body in it would mean intake had read the recording, which is the +// one thing the delivery path must not do. +func TestPointerLineCarriesNoContent(t *testing.T) { + var pointers bytes.Buffer + intake, _, _ := newTestIntake(t, nil, &pointers) + require.NoError(t, intake.ingest(context.Background(), testEvent(1), LanePoll)) + + var raw map[string]any + require.NoError(t, json.Unmarshal([]byte(strings.TrimSpace(pointers.String())), &raw)) + for _, forbidden := range []string{"title", "content", "body", "url", "app_url", "creator_name", "excerpt"} { + assert.NotContains(t, raw, forbidden) + } +} + +func TestPollServedIDAdvancesOnlyOnAPageBoundary(t *testing.T) { + polls := &scriptedPolls{pages: []eventfeed.PollPage{{ + Events: []eventfeed.Event{testEvent(17099838500)}, + Position: "p1", + }}} + intake, ledger, _ := newTestIntake(t, nil, nil) + intake.served = &servedPolls{inner: polls} + ctx := context.Background() + + live := testEvent(17099999999) + live.ActorType = "person" + visible := true + live.VisibleToClients = &visible + require.NoError(t, intake.ingest(ctx, live, LaneOf(live))) + intake.confirmPollServed(ctx, "some-other-position") + + served, err := ledger.LastPollServedID(ctx, intake.CheckpointKey()) + require.NoError(t, err) + assert.Zero(t, served, + "a live id is far ahead of the poll lane; re-entering at one skips everything inside the safety delay") + + page, err := intake.served.Poll(ctx, eventfeed.Cursor{}, eventfeed.Filters{}) + require.NoError(t, err) + served, err = ledger.LastPollServedID(ctx, intake.CheckpointKey()) + require.NoError(t, err) + assert.Zero(t, served, "a page served but not yet delivered records nothing") + + intake.confirmPollServed(ctx, page.Position) + served, err = ledger.LastPollServedID(ctx, intake.CheckpointKey()) + require.NoError(t, err) + assert.Equal(t, int64(17099838500), served) +} + +func TestOverflowIsOnDiskBeforeItIsAccepted(t *testing.T) { + intake, ledger, _ := newTestIntake(t, nil, nil) + ctx := context.Background() + + // One of the dropped ids is already in the ledger: it was dropped from the + // buffer, not from the connector. + _, err := ledger.RecordSeen(ctx, testEvent(17099838500), LaneLive) + require.NoError(t, err) + + disposition := intake.handleSignal(eventfeed.BufferOverflow{ + DroppedIDs: []int64{17099838500, 17099838501, 17099838502}, + DroppedCount: 3, + }) + assert.Equal(t, eventfeed.Accept, disposition) + + losses, err := ledger.OpenLosses(ctx) + require.NoError(t, err) + require.Len(t, losses, 1, "a loss that only ever lived in memory is one no restart could find") + assert.Equal(t, 3, losses[0].DroppedCount) + assert.Equal(t, int64(17099838500), losses[0].RepairSince, + "the walk enters one below the lowest MISSING id, and the feed's since is exclusive") + + missing, err := ledger.MissingIDs(ctx, losses[0].ID, LossMissing) + require.NoError(t, err) + assert.Equal(t, []int64{17099838501, 17099838502}, missing) + + neverLost, err := ledger.MissingIDs(ctx, losses[0].ID, LossNeverLost) + require.NoError(t, err) + assert.Equal(t, []int64{17099838500}, neverLost) +} + +func TestOverflowIsRefusedWhenItCannotBeWrittenDown(t *testing.T) { + intake, ledger, _ := newTestIntake(t, nil, nil) + require.NoError(t, ledger.Close()) + + disposition := intake.handleSignal(eventfeed.BufferOverflow{ + DroppedIDs: []int64{17099838501}, + DroppedCount: 1, + }) + assert.Equal(t, eventfeed.Terminate, disposition, + "accepting an incompleteness that was never recorded is worse than ending the feed") +} + +func TestOverflowOfIdsAlreadyHeldResolvesImmediately(t *testing.T) { + intake, ledger, _ := newTestIntake(t, nil, nil) + ctx := context.Background() + _, err := ledger.RecordSeen(ctx, testEvent(17099838500), LaneLive) + require.NoError(t, err) + + assert.Equal(t, eventfeed.Accept, intake.handleSignal(eventfeed.BufferOverflow{ + DroppedIDs: []int64{17099838500}, + DroppedCount: 1, + })) + + open, err := ledger.OpenLosses(ctx) + require.NoError(t, err) + assert.Empty(t, open, "nothing was lost, so there is nothing to walk for") +} + +func TestFeedGapRecordsTheEpochAndTheServedEntryClass(t *testing.T) { + intake, ledger, _ := newTestIntake(t, nil, nil) + ctx := context.Background() + + assert.Equal(t, eventfeed.Accept, intake.handleSignal(eventfeed.FeedGap{ + EpochAfterID: 17099838487, + ResumeURL: "https://3.basecampapi.com/2914079/events.json?since=17099838487", + })) + + gaps, err := ledger.Gaps(ctx) + require.NoError(t, err) + require.Len(t, gaps, 1) + assert.Equal(t, GapEpoch, gaps[0].Class) + require.NotNil(t, gaps[0].EpochAfterID) + assert.Equal(t, int64(17099838487), *gaps[0].EpochAfterID) + assert.Equal(t, EntryReplay, gaps[0].EntryClass) +} + +func TestFeedGapClassifiesAPresentEntryFromTheServedCursor(t *testing.T) { + intake, ledger, _ := newTestIntake(t, nil, nil) + + assert.Equal(t, eventfeed.Accept, intake.handleSignal(eventfeed.FeedGap{ + EpochAfterID: 17099838487, + ResumeURL: "https://3.basecampapi.com/2914079/events.json?since=now", + })) + + gaps, err := ledger.Gaps(context.Background()) + require.NoError(t, err) + require.Len(t, gaps, 1) + assert.Equal(t, EntryPresent, gaps[0].EntryClass) +} + +func TestFeedGapIsRefusedWhenItCannotBeRecorded(t *testing.T) { + intake, ledger, _ := newTestIntake(t, nil, nil) + require.NoError(t, ledger.Close()) + + assert.Equal(t, eventfeed.Terminate, intake.handleSignal(eventfeed.FeedGap{ + EpochAfterID: 17099838487, + ResumeURL: "https://3.basecampapi.com/2914079/events.json?since=17099838487", + }), "a gap nothing wrote down is a gap nothing will ever report") +} + +func TestLedgerRefusesAGapWhoseClassAndEpochDisagree(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + + _, err := ledger.RecordGap(ctx, Gap{DetectedAt: time.Now(), Class: GapEpoch}) + assert.Error(t, err, "an epoch gap without an epoch is the flattening this schema exists to refuse") + + epoch := int64(1) + _, err = ledger.RecordGap(ctx, Gap{DetectedAt: time.Now(), Class: GapRetention, EpochAfterID: &epoch}) + assert.Error(t, err, "a retention gap has no epoch to carry") +} + +func TestEntryClassOf(t *testing.T) { + assert.Equal(t, EntryPresent, entryClassOf("https://3.basecampapi.com/1/events.json?since=now")) + assert.Equal(t, EntryReplay, entryClassOf("https://3.basecampapi.com/1/events.json?since=17099838487")) + assert.Equal(t, EntryReplay, entryClassOf("https://3.basecampapi.com/1/events.json?since=0")) + assert.Equal(t, EntryUnknown, entryClassOf("https://3.basecampapi.com/1/events.json")) + assert.Equal(t, EntryUnknown, entryClassOf("https://3.basecampapi.com/1/events.json?since=soon")) + assert.Equal(t, EntryUnknown, entryClassOf("://not-a-url")) +} + +func TestReconciliationResumesOnStart(t *testing.T) { + polls := &scriptedPolls{pages: []eventfeed.PollPage{{ + Events: []eventfeed.Event{testEvent(17099838501)}, + Position: "walk-1", + }}} + intake, ledger, _ := newTestIntake(t, polls, nil) + ctx := context.Background() + + _, err := ledger.RecordLoss(ctx, []int64{17099838501}, time.Now(), time.Minute, eventfeed.Filters{}) + require.NoError(t, err) + + require.NoError(t, intake.resumeReconciliation(ctx)) + require.Eventually(t, func() bool { + open, err := ledger.OpenLosses(ctx) + return err == nil && len(open) == 0 + }, 5*time.Second, 5*time.Millisecond, + "a crash between the overflow and its repair is a delay, not a lost record") + + _, ok, err := ledger.Get(ctx, 17099838501) + require.NoError(t, err) + assert.True(t, ok) +} + +func TestNewRefusesAnIncompleteConfiguration(t *testing.T) { + ledger := newTestLedger(t) + queue, err := NewQueue(1, 2) + require.NoError(t, err) + + base := Options{ + Origin: "https://3.basecampapi.com", + AccountID: "2914079", + ConsumerNamespace: "connector-test", + Ledger: ledger, + Queue: queue, + Minter: stubMinter{}, + Polls: &scriptedPolls{}, + } + _, err = New(base) + require.NoError(t, err) + + noNamespace := base + noNamespace.ConsumerNamespace = "" + _, err = New(noNamespace) + assert.Error(t, err, "two connectors in one account must not share a checkpoint lineage") + + badFilters := base + badFilters.Filters = eventfeed.Filters{Types: []string{"comment.created,card.created"}} + _, err = New(badFilters) + assert.Error(t, err, "a filter set is validated before any wire attempt") +} + +func countLines(s string) int { + scanner := bufio.NewScanner(strings.NewReader(s)) + n := 0 + for scanner.Scan() { + if strings.TrimSpace(scanner.Text()) != "" { + n++ + } + } + return n +} + +// The card's acceptance number: a burst of a thousand events is seen within a +// minute. The bound here is far tighter than a minute on purpose — the point +// is to catch a regression that makes intake per-event expensive (a fsync per +// row, a read per pointer), not to measure the machine. +func TestABurstOfAThousandEventsIsAbsorbedQuickly(t *testing.T) { + var pointers bytes.Buffer + intake, ledger, queue := newTestIntake(t, nil, &pointers) + ctx := context.Background() + + start := time.Now() + for id := int64(17099838500); id < 17099838500+1000; id++ { + require.NoError(t, intake.ingest(ctx, testEvent(id), LanePoll)) + } + elapsed := time.Since(start) + + assert.Equal(t, 1000, queue.Depth()) + assert.Equal(t, 1000, countLines(pointers.String())) + seen, err := ledger.CountInState(ctx, StateSeen) + require.NoError(t, err) + assert.Equal(t, 1000, seen) + assert.Less(t, elapsed, 30*time.Second, "intake is the only work on the feed's delivery path") + t.Logf("1,000 events through intake in %s", elapsed) +} diff --git a/internal/connector/invariants_test.go b/internal/connector/invariants_test.go new file mode 100644 index 000000000..596262e02 --- /dev/null +++ b/internal/connector/invariants_test.go @@ -0,0 +1,458 @@ +package connector + +import ( + "bytes" + "context" + "errors" + "log/slog" + "path/filepath" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" +) + +// One test per invariant the design review found unenforced or untested. The +// invariant names match the review's list (A positions, C repair, D +// classification, E lifecycle, F queue, G lock, H confidentiality). + +// H1, repair side: the walk's logs render a failure's kind, never its text. +func TestInvariantH1RepairLogsRenderNoFailureText(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, clock.at, time.Minute, eventfeed.Filters{}) + require.NoError(t, err) + + var logs bytes.Buffer + polls := &scriptedPolls{errs: []error{ + errors.New("position=SECRET-POSITION"), + &eventfeed.PollError{Kind: eventfeed.PollUnrecoverable, Err: errors.New("next=SECRET-NEXT")}, + }} + walker, _ := newTestWalker(t, ledger, polls, clock) + walker.log = slog.New(slog.NewTextHandler(&logs, nil)) + require.NoError(t, walker.reconcile(ctx, loss)) + + assert.NotContains(t, logs.String(), "SECRET") +} + +type failingPositions struct{} + +func (failingPositions) LastPollServedID(context.Context, eventfeed.CheckpointKey) (int64, error) { + return 0, errors.New("disk I/O error") +} + +func (failingPositions) LineagePollServedID(context.Context, eventfeed.CheckpointKey) (int64, error) { + return 0, errors.New("disk I/O error") +} + +// A3: when the safe re-entry after a refused position cannot be read, the +// connection ends rather than letting the package enter at the present. +func TestInvariantA3AnUnreadableReentryEndsTheFeedInsteadOfEnteringAtThePresent(t *testing.T) { + ledger := newTestLedger(t) + intake, transport, minter, polls := newFeedIntake(t, ledger, Options{}) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + require.NoError(t, ledger.Save(ctx, intake.CheckpointKey(), "position-the-server-will-refuse")) + intake.positions = failingPositions{} + + minter.ScriptTicket(ticket()) + polls.ScriptError(&eventfeed.PollError{Kind: eventfeed.PollPositionInvalid}) + polls.ScriptPage(eventfeed.PollPage{Position: "present-entry"}) + + done := runInBackground(ctx, t, intake) + subscribedConn(t, transport) + + err := awaitReturn(t, done, "a feed whose safe re-entry cannot be read must end") + assert.Error(t, err) + for _, call := range polls.Calls() { + assert.NotEqual(t, "now", call.Cursor.Since, "no poll may enter at the present") + } +} + +// G1: one lock per canonical account and agent. +func TestInvariantG1OneLockPerCanonicalAccount(t *testing.T) { + dir := t.TempDir() + first, err := AcquireInstanceLock(dir, "2914079", 52007412, time.Now()) + require.NoError(t, err) + t.Cleanup(func() { _ = first.Release() }) + + _, err = AcquireInstanceLock(dir, "02914079", 52007412, time.Now()) + assert.Error(t, err, "02914079 is the same account and must meet the same lock") + _, err = AcquireInstanceLock(dir, "0", 52007412, time.Now()) + assert.Error(t, err) +} + +// C2: a 410 on a pass that has not yet followed that 410's resume is +// followed, whatever the walk's entry happens to equal. Ids above the fence +// stay recoverable. +func TestInvariantC2A410OnAStoredPositionIsFollowedEvenAtTheFence(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + loss, err := ledger.RecordLoss(ctx, []int64{100, 200}, clock.at, 10*time.Minute, eventfeed.Filters{}) + require.NoError(t, err) + _, err = ledger.MarkUnrecoveredThrough(ctx, loss.ID, 150) + require.NoError(t, err) + require.NoError(t, ledger.SetRepairEntry(ctx, loss.ID, 150)) + require.NoError(t, ledger.SaveRepairCursor(ctx, loss.ID, "position-above-the-fence")) + loss.RepairSince = 150 + loss.RepairCursor = "position-above-the-fence" + + polls := fencedPolls{resume: "https://3.basecampapi.com/2914079/events.json?since=150"} + walker, _ := newTestWalker(t, ledger, polls, clock) + require.NoError(t, walker.reconcile(ctx, loss)) + + recovered, err := ledger.MissingIDs(ctx, loss.ID, LossRecovered) + require.NoError(t, err) + assert.Equal(t, []int64{200}, recovered) +} + +// fencedPolls answers every stored position with the epoch's 410, and serves +// the event above the fence only from the resume. +type fencedPolls struct{ resume string } + +func (f fencedPolls) Poll(_ context.Context, cursor eventfeed.Cursor, _ eventfeed.Filters) (eventfeed.PollPage, error) { + if cursor.PageURL == f.resume || cursor.Since == "150" { + return eventfeed.PollPage{Events: []eventfeed.Event{testEvent(200)}, Position: "above"}, nil + } + return eventfeed.PollPage{}, &eventfeed.PollError{Kind: eventfeed.PollGone, EpochAfterID: 150, ResumeURL: f.resume} +} + +// F2: warning edges settle on the true state under concurrent offers and +// takes. +func TestInvariantF2WarningEdgesSettleOnTheTrueState(t *testing.T) { + for range 50 { + queue, err := NewQueue(1, 64) + require.NoError(t, err) + var warns, recovers atomic.Int32 + queue.OnWarn = func(int) { warns.Add(1) } + queue.OnRecover = func(int) { recovers.Add(1) } + + ctx := context.Background() + var wg sync.WaitGroup + for g := range 8 { + wg.Add(1) + go func() { + defer wg.Done() + for i := range 200 { + require.NoError(t, queue.Offer(ctx, int64(g*1000+i))) + _, err := queue.Take(ctx) + require.NoError(t, err) + } + }() + } + wg.Wait() + require.Zero(t, queue.Depth()) + require.Equal(t, warns.Load(), recovers.Load(), "every warning is answered once the backlog is gone") + } +} + +// F1: while intake is paused on a full queue, the page it is in the middle of +// is not checkpointed. +func TestInvariantF1APausedFeedDoesNotMoveTheCheckpoint(t *testing.T) { + ledger := newTestLedger(t) + queue, err := NewQueue(1, 1) + require.NoError(t, err) + intake, transport, minter, polls := newFeedIntake(t, ledger, Options{}) + intake.queue = queue + intake.opts.Queue = queue + minter.ScriptTicket(ticket()) + polls.ScriptPage(eventfeed.PollPage{Events: []eventfeed.Event{testEvent(500), testEvent(501)}, Position: "after-both"}) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := runInBackground(ctx, t, intake) + subscribedConn(t, transport) + + require.Eventually(t, queue.Paused, 5*time.Second, time.Millisecond) + time.Sleep(50 * time.Millisecond) + _, ok, err := ledger.Load(ctx, intake.CheckpointKey()) + require.NoError(t, err) + assert.False(t, ok, "a crash now must resume from before the page, not after it") + + _, err = queue.Take(ctx) + require.NoError(t, err) + require.Eventually(t, func() bool { + position, ok, err := ledger.Load(ctx, intake.CheckpointKey()) + return err == nil && ok && position == "after-both" + }, 5*time.Second, 5*time.Millisecond) + + cancel() + awaitReturn(t, done, "Run should return on shutdown") +} + +// E2: a reconnect asked for between connections is answered by the next +// connection, not carried into it to turn its terminal error into a silent +// reconnect. +func TestInvariantE2AStaleReconnectDoesNotSwallowATerminalError(t *testing.T) { + ledger := newTestLedger(t) + intake, _, minter, _ := newFeedIntake(t, ledger, Options{}) + for range 4 { + minter.ScriptError(&eventfeed.MintError{Kind: eventfeed.MintUnrecoverable, Err: errors.New("gone")}) + } + // Asked for while no connection was running: the next connection IS the + // answer to it. + intake.requestReconnect() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + err := awaitReturn(t, runInBackground(ctx, t, intake), "the terminal error must end Run") + assert.Error(t, err) + assert.Equal(t, 1, minter.Calls(), + "a latch carried into the connection turns its terminal error into a reconnect") +} + +// E3: the listing the lister gives is what decides a change. A bucket learned +// from an arriving event is provisional and never counts as one, and it is +// revoked when a listing omits it. +func TestInvariantE3MembershipComparison(t *testing.T) { + intake, _, _ := newTestIntake(t, nil, nil) + intake.opts.Membership = &flakyMembership{buckets: []int64{1, 2}} + + require.False(t, intake.adoptListing([]int64{1, 2}), "the first listing is a baseline") + intake.noteBucket(9) + assert.False(t, intake.adoptListing([]int64{1, 2}), "a learned bucket the listing omits is not a change") + + intake.mu.Lock() + held := intake.learned[9] + intake.mu.Unlock() + assert.False(t, held, "and that listing revoked it") + + assert.True(t, intake.adoptListing([]int64{1}), "a listed bucket dropping off is a change") + assert.True(t, intake.adoptListing([]int64{1, 2}), "and its return is a change") +} + +// H2 availability: two processes opening one fresh ledger both get it. +func TestInvariantH2ConcurrentFreshOpensAllSucceed(t *testing.T) { + path := filepath.Join(t.TempDir(), "state", "connector.db") + var wg sync.WaitGroup + errs := make(chan error, 32) + for range 32 { + wg.Add(1) + go func() { + defer wg.Done() + ledger, err := OpenLedger(path) + if err == nil { + err = ledger.Close() + } + errs <- err + }() + } + wg.Wait() + close(errs) + for err := range errs { + assert.NoError(t, err) + } +} + +// E4: terminal means terminal. A completed or discarded record that could move +// back into the working states could be dispatched a second time — and once +// DropContent has taken its payload, requeued as work with nothing in it. +func TestInvariantE4TerminalRecordsHaveNoWayBack(t *testing.T) { + active := []RecordState{StateSeen, StateAdmitted, StateQueued, StateBlocked, StateDispatched} + for _, terminal := range []RecordState{StateCompleted, StateDiscarded} { + for _, target := range append(active, terminalPeer(terminal)) { + t.Run(string(terminal)+" to "+string(target), func(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + require.NoError(t, reachTerminal(t, ledger, 1, terminal)) + + reason := "" + if target == StateBlocked || target == StateDiscarded { + reason = "a reason" + } + err := ledger.SetState(ctx, 1, target, reason) + + require.Error(t, err) + assert.ErrorIs(t, err, ErrNotATransition) + record, ok, getErr := ledger.Get(ctx, 1) + require.NoError(t, getErr) + require.True(t, ok) + assert.Equal(t, terminal, record.State, "the record stayed where it was") + }) + } + } +} + +// Writing the state a record already has is a repeat, not a move: a retry +// after a crash is not an error, and the record does not leave its state. +func TestInvariantE4TerminalRecordsTolerateARepeat(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + require.NoError(t, reachTerminal(t, ledger, 1, StateCompleted)) + + require.NoError(t, ledger.SetState(ctx, 1, StateCompleted, "")) + + record, ok, err := ledger.Get(ctx, 1) + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, StateCompleted, record.State) +} + +// The refusal is the database's too, so anything that ever writes to this file +// meets it — not only this package's own SetState. +func TestInvariantE4TheDatabaseRefusesLeavingATerminalState(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + require.NoError(t, reachTerminal(t, ledger, 1, StateDiscarded)) + + _, err := ledger.db.ExecContext(ctx, `UPDATE events SET state = 'seen' WHERE id = 1`) + + require.Error(t, err) + assert.Contains(t, err.Error(), "terminal record") +} + +// A record that does not exist is told apart from a transition that is not +// allowed: one is a missing row, the other a refusal. +func TestInvariantE4AMissingRecordIsNotARefusedTransition(t *testing.T) { + ledger := newTestLedger(t) + + err := ledger.SetState(context.Background(), 404, StateAdmitted, "") + + assert.ErrorIs(t, err, ErrNoSuchRecord) + assert.NotErrorIs(t, err, ErrNotATransition) +} + +// terminalPeer is the other terminal state, so the pairs cover completed to +// discarded and back. +func terminalPeer(state RecordState) RecordState { + if state == StateCompleted { + return StateDiscarded + } + return StateCompleted +} + +// reachTerminal walks a fresh record to a terminal state along the lifecycle's +// own edges. +func reachTerminal(t *testing.T, ledger *Ledger, id int64, terminal RecordState) error { + t.Helper() + ctx := context.Background() + if _, err := ledger.RecordSeen(ctx, testEvent(id), LanePoll); err != nil { + return err + } + if terminal == StateDiscarded { + return ledger.SetState(ctx, id, StateDiscarded, "untrusted_author") + } + if err := ledger.SetState(ctx, id, StateAdmitted, ""); err != nil { + return err + } + if err := ledger.SetState(ctx, id, StateDispatched, ""); err != nil { + return err + } + return ledger.SetState(ctx, id, StateCompleted, "") +} + +// A2: a cursor older than the feed's current epoch is never usable. A 410 with +// an epoch says the history below it is gone for good, so the position the +// walk was holding is not merely superseded — it is refused by the server +// forever, and a pass that ends for some other reason must not hand it back. +func TestInvariantA2APreEpochCursorIsNeverUsedAgain(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + // The missing id is above the epoch, so the walk still has something to + // serve after the fence and follows the resume. + loss, err := ledger.RecordLoss(ctx, []int64{17099838700}, clock.at, time.Hour, eventfeed.Filters{}) + require.NoError(t, err) + require.NoError(t, ledger.SaveRepairCursor(ctx, loss.ID, "PRE-EPOCH-POSITION")) + loss.RepairCursor = "PRE-EPOCH-POSITION" + + const epoch = int64(17099838600) + polls := &scriptedPolls{errs: []error{ + // The stored cursor is below the epoch: the fence. + &eventfeed.PollError{ + Kind: eventfeed.PollGone, + EpochAfterID: epoch, + ResumeURL: "https://3.basecampapi.com/2914079/events.json?since=17099838600", + Err: errors.New("gone"), + }, + // The resume then fails transiently — a connection reset, not a + // verdict — and the pass ends. + errors.New("connection reset"), + }} + walker, _ := newTestWalker(t, ledger, polls, clock) + + cursor, err := walker.walk(ctx, &loss) + require.NoError(t, err) + + assert.Empty(t, cursor, "the pass must not carry the refused position back to its caller") + assert.Empty(t, loss.RepairCursor) + stored, err := ledger.OpenLosses(ctx) + require.NoError(t, err) + require.Len(t, stored, 1) + assert.Empty(t, stored[0].RepairCursor, "the pre-epoch cursor is gone from the ledger too") + assert.Equal(t, epoch, stored[0].RepairSince) + + // The next pass enters at the epoch, never at the refused position. + polls.errs = nil + _, err = walker.walk(ctx, &stored[0]) + require.NoError(t, err) + entered := polls.cursors[len(polls.cursors)-1] + assert.Equal(t, "17099838600", entered.Since) + // The first poll of the first pass is where the refusal was discovered. + // Nothing after it may name that position again. + for _, seen := range polls.cursors[1:] { + assert.NotEqual(t, "PRE-EPOCH-POSITION", seen.Position, "no pass may re-enter at the refused position") + } +} + +// The lifecycle carries the edges the later cards actually commit, so neither +// has to write state around SetState to make its own contract work. +func TestInvariantE4TheLifecycleCarriesTheEdgesLaterCardsCommit(t *testing.T) { + // Admission commits an admitted verdict AS queued when the conversation + // is live, so the record goes from seen to queued in one write. + t.Run("seen to queued", func(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + _, err := ledger.RecordSeen(ctx, testEvent(1), LanePoll) + require.NoError(t, err) + + require.NoError(t, ledger.SetState(ctx, 1, StateQueued, "")) + + record, ok, err := ledger.Get(ctx, 1) + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, StateQueued, record.State) + }) + + // A dispatched record whose worker never started has its exposure + // withdrawn and returns to admitted. + t.Run("dispatched back to admitted", func(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + _, err := ledger.RecordSeen(ctx, testEvent(1), LanePoll) + require.NoError(t, err) + require.NoError(t, ledger.SetState(ctx, 1, StateAdmitted, "")) + require.NoError(t, ledger.SetState(ctx, 1, StateDispatched, "")) + + require.NoError(t, ledger.SetState(ctx, 1, StateAdmitted, "")) + + record, ok, err := ledger.Get(ctx, 1) + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, StateAdmitted, record.State) + }) +} + +// updated_at is the retention clock. Writing the state a record already has is +// a repeat, and a repeat must not restart the window on a finished record. +func TestInvariantE4ARepeatDoesNotRestartRetention(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + at := time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC) + ledger.now = func() time.Time { return at } + require.NoError(t, reachTerminal(t, ledger, 1, StateCompleted)) + + ledger.now = func() time.Time { return at.Add(90 * 24 * time.Hour) } + require.NoError(t, ledger.SetState(ctx, 1, StateCompleted, "")) + + dropped, err := ledger.DropContent(ctx, at.Add(time.Hour), at.Add(time.Hour)) + require.NoError(t, err) + assert.Equal(t, 1, dropped, "the repeat must not have pushed the record's retention forward") +} diff --git a/internal/connector/ledger.go b/internal/connector/ledger.go new file mode 100644 index 000000000..0a9589bd3 --- /dev/null +++ b/internal/connector/ledger.go @@ -0,0 +1,343 @@ +package connector + +import ( + "context" + "database/sql" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "modernc.org/sqlite" // database/sql driver "sqlite", pure Go: no cgo on any of the five release targets. + sqlite3 "modernc.org/sqlite/lib" + + "github.com/basecamp/basecamp-cli/internal/connector/setup" +) + +// isInMemory reports a path SQLite would read as its in-memory database +// rather than a file. +func isInMemory(path string) bool { + trimmed := strings.TrimPrefix(path, "file:") + return trimmed == ":memory:" || strings.HasPrefix(trimmed, ":memory:?") || trimmed == "" +} + +// RecordState is where an event sits in the ledger's lifecycle. +// +// Intake only ever writes StateSeen. The rest of the vocabulary is declared +// here because the states are one lifecycle, and a store that cannot name the +// state a later card writes cannot recover it on start either. +type RecordState string + +const ( + // StateSeen is a pointer intake wrote and nothing has judged yet. Every + // seen record re-runs the gate on start. + StateSeen RecordState = "seen" + // StateAdmitted passed the gate and awaits dispatch. + StateAdmitted RecordState = "admitted" + // StateQueued waits behind another event on its conversation. + StateQueued RecordState = "queued" + // StateBlocked is retained and retried: a reason, never a transport + // failure dressed up as a verdict. + StateBlocked RecordState = "blocked" + // StateDispatched was handed to a worker. + StateDispatched RecordState = "dispatched" + // StateCompleted is terminal with an outcome. + StateCompleted RecordState = "completed" + // StateDiscarded is terminal with a verified verdict. + StateDiscarded RecordState = "discarded" +) + +// Lane names which lane first served an event. It is diagnostic: dedupe is by +// id, and the same event ordinarily arrives on both. +type Lane string + +const ( + // LaneLive is the WebSocket. + LaneLive Lane = "live" + // LanePoll is the catch-up or streaming poll walk. + LanePoll Lane = "poll" + // LaneRepair is a repair walk after an overflow — its own cursor, never + // the feed's. + LaneRepair Lane = "repair" +) + +// Ledger is the connector's durable memory: the dedupe authority, the feed +// position, and the record of what was lost. +// +// It is a SQLite file rather than a set in memory because every promise the +// connector makes about a crash rests on the answer to "have I seen this id +// before?" surviving the crash. +type Ledger struct { + db *sql.DB + now func() time.Time +} + +// OpenLedger opens (creating if absent) the ledger at path and brings its +// schema up to date. +func OpenLedger(path string) (*Ledger, error) { + if path == "" { + return nil, errors.New("connector: ledger path is required") + } + if isInMemory(path) { + // SQLite's in-memory URI accepts every write and loses it on close. + // The ledger's whole promise is that a crash is a delay. + return nil, fmt.Errorf("connector: ledger path %q names SQLite's in-memory database, which is not durable", path) + } + if strings.ContainsAny(path, "?#%") { + // The driver reads the path as a URI; these would be taken as its + // query, fragment or an escape, and open some other file. + return nil, fmt.Errorf("connector: ledger path %q contains a character the SQLite URI cannot carry (?, # or %%)", path) + } + if err := securePath(path); err != nil { + return nil, err + } + + // _txlock=immediate takes the write lock when a transaction opens rather + // than on its first write. Without it two connectors racing on one file + // can both start, both read, and one is refused at COMMIT with the work + // already done. + dsn := "file:" + path + "?_pragma=busy_timeout(5000)&_pragma=journal_mode(WAL)&_pragma=foreign_keys(1)&_txlock=immediate" + db, err := sql.Open("sqlite", dsn) + if err != nil { + return nil, fmt.Errorf("connector: open ledger: %w", err) + } + // One writer. SQLite serializes writers anyway, and a pool merely turns + // that serialization into SQLITE_BUSY under load. + db.SetMaxOpenConns(1) + + l := &Ledger{db: db, now: time.Now} + if err := retryBusy(func() error { return l.migrate(context.Background()) }); err != nil { + _ = db.Close() + return nil, err + } + // The WAL and shared-memory sidecars exist now and were created under the + // process umask. The private directory already keeps other users out; + // tightening them too costs nothing. + for _, sidecar := range []string{path + "-wal", path + "-shm"} { + if err := os.Chmod(sidecar, 0o600); err != nil && !os.IsNotExist(err) { + _ = db.Close() + return nil, fmt.Errorf("connector: secure ledger sidecar: %w", err) + } + } + return l, nil +} + +// retryBusy retries fn while SQLite reports the database busy, for up to five +// seconds. +// +// The busy timeout covers ordinary contention, but not all of it: switching a +// fresh database into WAL mode takes a lock the busy handler is not consulted +// for, so two processes opening one new ledger at once — `status` beside a +// starting connector — can get SQLITE_BUSY immediately. The migration is +// idempotent, so trying again is safe. +func retryBusy(fn func() error) error { + deadline := time.Now().Add(5 * time.Second) + for { + err := fn() + if err == nil || !isBusy(err) || time.Now().After(deadline) { + return err + } + time.Sleep(20 * time.Millisecond) + } +} + +func isBusy(err error) bool { + var sqliteErr *sqlite.Error + if !errors.As(err, &sqliteErr) { + return false + } + // The primary result code, without the extended bits. + switch sqliteErr.Code() & 0xff { + case sqlite3.SQLITE_BUSY, sqlite3.SQLITE_LOCKED: + return true + } + return false +} + +// securePath makes the ledger private or refuses it. +// +// The ledger holds feed positions — signed tokens that resume the account's +// feed — and every event's metadata, so it is a credential file and is +// treated as one. The check is the same one the connector's trust file and +// instance lock get: every directory on the way must be this user's own and +// unwritable by anyone else, and the ledger itself is opened without +// following symlinks and inspected through that descriptor rather than by +// name. Validating the name would validate whatever the name pointed at when +// it was asked, which is not necessarily what SQLite then opens. +// +// A file or directory that already exists with looser permissions is refused +// rather than tightened: something else chose those permissions, and silently +// changing them could break it or hide that the ledger was exposed. +// +// The check cannot be made on a platform without POSIX owners and modes, and +// a ledger whose privacy cannot be established is refused there rather than +// opened — the same way setup refuses to write a trust file it cannot vouch +// for. +func securePath(path string) error { + if err := setup.EnsurePrivateFile(path); err != nil { + return fmt.Errorf("connector: secure the ledger: %w", err) + } + // One rule of the ledger's own, beyond what a trust file needs: its + // directory must be 0700, not merely unwritable by others. SQLite writes + // -wal and -shm beside the database, and a directory other users can read + // is one whose entries they can list. The directory is already known not + // to be a symlink, so Lstat here inspects the directory itself. + dir := filepath.Dir(path) + info, err := os.Lstat(dir) + if err != nil { + return fmt.Errorf("connector: inspect ledger directory: %w", err) + } + if perm := info.Mode().Perm(); perm&0o077 != 0 { + return fmt.Errorf("connector: ledger directory %s is readable by other users (mode %04o); it must be 0700", dir, perm) + } + return nil +} + +// Close releases the ledger's handle. +func (l *Ledger) Close() error { return l.db.Close() } + +// migrations are applied in order, each exactly once. A migration is never +// edited after it ships: the ledger outlives the binary that created it. +var migrations = []string{ + ` +CREATE TABLE events ( + id INTEGER PRIMARY KEY, + state TEXT NOT NULL, + reason TEXT NOT NULL DEFAULT '', + lane TEXT NOT NULL, + event_type TEXT NOT NULL, + kind TEXT NOT NULL, + action TEXT NOT NULL, + bucket_id INTEGER NOT NULL, + creator_id INTEGER NOT NULL, + performed_by_id INTEGER, + recording_id INTEGER NOT NULL, + details BLOB, + actor_type TEXT NOT NULL DEFAULT '', + visible_to_clients INTEGER, + created_at TEXT NOT NULL, + seen_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + content_dropped INTEGER NOT NULL DEFAULT 0 +); +CREATE INDEX events_state_id ON events (state, id); + +CREATE TABLE checkpoints ( + flat_key TEXT PRIMARY KEY, + lineage TEXT NOT NULL, + position TEXT NOT NULL, + last_poll_served_id INTEGER NOT NULL DEFAULT 0, + updated_at TEXT NOT NULL +); +CREATE INDEX checkpoints_lineage ON checkpoints (lineage); + +CREATE TABLE losses ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + detected_at TEXT NOT NULL, + dropped_count INTEGER NOT NULL, + repair_since INTEGER NOT NULL, + repair_cursor TEXT NOT NULL DEFAULT '', + deadline_at TEXT NOT NULL, + resolved_at TEXT +); + +CREATE TABLE loss_ids ( + loss_id INTEGER NOT NULL REFERENCES losses (id) ON DELETE CASCADE, + event_id INTEGER NOT NULL, + state TEXT NOT NULL, + PRIMARY KEY (loss_id, event_id) +); +CREATE INDEX loss_ids_event ON loss_ids (event_id, state); + +CREATE TABLE gaps ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + detected_at TEXT NOT NULL, + class TEXT NOT NULL, + epoch_after_id INTEGER, + entry_class TEXT NOT NULL DEFAULT '', + note TEXT NOT NULL DEFAULT '' +); +`, + // Migration 2. A loss is repaired under the filter set it was recorded + // with: a connector restarted with different filters must not walk the + // wrong lane for it and condemn events the original filters would have + // served. Migrations only ever append, so an existing ledger takes this + // one and its open losses carry an empty set, which reads as "the + // connector's own". + `ALTER TABLE losses ADD COLUMN filters TEXT NOT NULL DEFAULT ''`, + // Migration 3. Terminal means terminal, in the database and not only in + // the code that writes to it. A completed or discarded record that could + // be moved back into the working states could be dispatched a second + // time, or — once its payload has been dropped and only the tombstone + // remains — requeued as work with nothing in it. The Go side refuses + // every edge the lifecycle does not have; this refuses the two that + // matter to anything that ever writes to this file. + ` +CREATE TRIGGER events_terminal_is_terminal +BEFORE UPDATE OF state ON events +WHEN OLD.state IN ('completed', 'discarded') AND NEW.state <> OLD.state +BEGIN + SELECT RAISE(ABORT, 'a terminal record cannot change state'); +END; +`, +} + +func (l *Ledger) migrate(ctx context.Context) error { + if _, err := l.db.ExecContext(ctx, `CREATE TABLE IF NOT EXISTS schema_migrations ( + version INTEGER PRIMARY KEY, + applied_at TEXT NOT NULL +)`); err != nil { + return fmt.Errorf("connector: create migration table: %w", err) + } + + for i := range migrations { + version := i + 1 + // The version is read inside the migration's own transaction, which + // takes the write lock as it opens: two processes opening a fresh + // ledger at once — `status` beside a starting connector — must not + // both decide migration 1 is theirs to apply. + tx, err := l.db.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("connector: begin migration %d: %w", version, err) + } + var applied int + if err := tx.QueryRowContext(ctx, `SELECT COALESCE(MAX(version), 0) FROM schema_migrations`).Scan(&applied); err != nil { + _ = tx.Rollback() + return fmt.Errorf("connector: read schema version: %w", err) + } + if applied >= version { + _ = tx.Rollback() + continue + } + if _, err := tx.ExecContext(ctx, migrations[i]); err != nil { + _ = tx.Rollback() + return fmt.Errorf("connector: apply migration %d: %w", version, err) + } + if _, err := tx.ExecContext(ctx, `INSERT INTO schema_migrations (version, applied_at) VALUES (?, ?)`, version, l.timestamp()); err != nil { + _ = tx.Rollback() + return fmt.Errorf("connector: record migration %d: %w", version, err) + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("connector: commit migration %d: %w", version, err) + } + } + return nil +} + +// SchemaVersion reports the highest applied migration. +func (l *Ledger) SchemaVersion(ctx context.Context) (int, error) { + var version int + err := l.db.QueryRowContext(ctx, `SELECT COALESCE(MAX(version), 0) FROM schema_migrations`).Scan(&version) + return version, err +} + +func (l *Ledger) timestamp() string { return stamp(l.now()) } + +// ledgerTime is the one format every ledger timestamp is stored in: UTC, with +// all nine fractional digits. Fixed width is what makes SQLite's text +// comparison agree with time order. time.RFC3339Nano trims trailing zeros, so +// "12:00:00Z" would sort after the later "12:00:00.5Z" and retention would +// keep a record past its window. +const ledgerTime = "2006-01-02T15:04:05.000000000Z07:00" diff --git a/internal/connector/ledger_checkpoint.go b/internal/connector/ledger_checkpoint.go new file mode 100644 index 000000000..2a973dc03 --- /dev/null +++ b/internal/connector/ledger_checkpoint.go @@ -0,0 +1,141 @@ +package connector + +import ( + "context" + "database/sql" + "errors" + "fmt" + + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" +) + +// Ledger implements the feed's checkpoint seam. +var _ eventfeed.CheckpointStore = (*Ledger)(nil) + +// Load returns the stored position for key. It runs once, before the first +// mint, and a store failure is a failure to start rather than a silent entry +// at the present: re-entering at the head after a crash drops everything the +// feed committed while we were down. +func (l *Ledger) Load(ctx context.Context, key eventfeed.CheckpointKey) (string, bool, error) { + var position string + err := l.db.QueryRowContext(ctx, + `SELECT position FROM checkpoints WHERE flat_key = ? AND position <> ''`, key.FlatKey()).Scan(&position) + switch { + case errors.Is(err, sql.ErrNoRows): + // No row, or a row whose position was dropped by a 409 while its + // last poll-served id was kept. Both mean "no position to resume"; + // an empty string reported as present would be sent to the server + // as a position and refused as malformed. + return "", false, nil + case err != nil: + return "", false, fmt.Errorf("connector: load checkpoint: %w", err) + } + return position, true, nil +} + +// Save durably records position under key. +// +// The feed calls this only for a poll page, and only once that page's events +// have been accepted by the consumer — which, here, means they are already +// rows in the events table. A live event id never reaches this method, and +// must not: a live id runs ahead of the poll lane's safety delay, so a +// position taken from one would skip everything inside that window. +func (l *Ledger) Save(ctx context.Context, key eventfeed.CheckpointKey, position string) error { + _, err := l.db.ExecContext(ctx, ` +INSERT INTO checkpoints (flat_key, lineage, position, updated_at) VALUES (?, ?, ?, ?) +ON CONFLICT (flat_key) DO UPDATE SET position = excluded.position, updated_at = excluded.updated_at`, + key.FlatKey(), lineageOf(key), position, l.timestamp()) + if err != nil { + return fmt.Errorf("connector: save checkpoint: %w", err) + } + return nil +} + +// NotePollServed advances the last poll-served event id for key. +// +// This is tracked apart from the position, and apart from the last id +// delivered, because all three answer different questions. The position is an +// opaque signed token, so nothing can be compared to it or derived from it. The +// last delivered id is usually a live id, thirty-odd seconds ahead of the poll +// lane. Only the last poll-served id is a safe place to re-enter the feed +// after a 409 or a rejected position — anything ahead of it skips events the +// poll lane had not served yet. +// +// It only ever moves forward. +func (l *Ledger) NotePollServed(ctx context.Context, key eventfeed.CheckpointKey, eventID int64) error { + _, err := l.db.ExecContext(ctx, ` +INSERT INTO checkpoints (flat_key, lineage, position, last_poll_served_id, updated_at) VALUES (?, ?, '', ?, ?) +ON CONFLICT (flat_key) DO UPDATE SET + last_poll_served_id = MAX(checkpoints.last_poll_served_id, excluded.last_poll_served_id), + updated_at = excluded.updated_at`, + key.FlatKey(), lineageOf(key), eventID, l.timestamp()) + if err != nil { + return fmt.Errorf("connector: note poll-served id: %w", err) + } + return nil +} + +// LastPollServedID returns the last id the poll lane served under key, or zero +// if it has served none. +func (l *Ledger) LastPollServedID(ctx context.Context, key eventfeed.CheckpointKey) (int64, error) { + var id int64 + err := l.db.QueryRowContext(ctx, + `SELECT last_poll_served_id FROM checkpoints WHERE flat_key = ?`, key.FlatKey()).Scan(&id) + switch { + case errors.Is(err, sql.ErrNoRows): + return 0, nil + case err != nil: + return 0, fmt.Errorf("connector: read last poll-served id: %w", err) + } + return id, nil +} + +// LineagePollServedID returns the highest id the poll lane has served to this +// consumer under ANY filter set: same origin, account and namespace. +// +// It is the re-entry for a filter change. The new digest holds no position of +// its own, and entering it at the present would skip everything committed +// since the old digest's walk stopped. Events a narrower old filter excluded +// before this id are not recovered by widening — the spec says so, and so does +// this comment — but nothing after it is skipped. +func (l *Ledger) LineagePollServedID(ctx context.Context, key eventfeed.CheckpointKey) (int64, error) { + var id int64 + err := l.db.QueryRowContext(ctx, + `SELECT COALESCE(MAX(last_poll_served_id), 0) FROM checkpoints WHERE lineage = ?`, lineageOf(key)).Scan(&id) + if err != nil { + return 0, fmt.Errorf("connector: read lineage poll-served id: %w", err) + } + return id, nil +} + +// LineageHasPosition reports whether this consumer holds a position under any +// filter set. +func (l *Ledger) LineageHasPosition(ctx context.Context, key eventfeed.CheckpointKey) (bool, error) { + var n int + err := l.db.QueryRowContext(ctx, + `SELECT COUNT(*) FROM checkpoints WHERE lineage = ? AND position <> ''`, lineageOf(key)).Scan(&n) + if err != nil { + return false, fmt.Errorf("connector: read lineage positions: %w", err) + } + return n > 0, nil +} + +// lineageOf is the checkpoint identity without its filter digest. +func lineageOf(key eventfeed.CheckpointKey) string { + key.FilterKey = "" + return key.FlatKey() +} + +// ForgetPosition drops the held position for key while keeping the last +// poll-served id — the 409 path. The server refused the position because it +// was minted for a different filter set; the id the poll lane had reached is +// still true, and is where the new digest re-enters. +func (l *Ledger) ForgetPosition(ctx context.Context, key eventfeed.CheckpointKey) error { + _, err := l.db.ExecContext(ctx, + `UPDATE checkpoints SET position = '', updated_at = ? WHERE flat_key = ?`, + l.timestamp(), key.FlatKey()) + if err != nil { + return fmt.Errorf("connector: forget position: %w", err) + } + return nil +} diff --git a/internal/connector/ledger_events.go b/internal/connector/ledger_events.go new file mode 100644 index 000000000..1867f7213 --- /dev/null +++ b/internal/connector/ledger_events.go @@ -0,0 +1,365 @@ +package connector + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "slices" + "strings" + "time" + + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" +) + +// Record is one row of the events table. +type Record struct { + ID int64 + State RecordState + Reason string + Lane Lane + EventType string + Kind string + Action string + BucketID int64 + CreatorID int64 + PerformedByID *int64 + RecordingID int64 + Details json.RawMessage + ActorType string + VisibleToClients *bool + CreatedAt time.Time + SeenAt time.Time + UpdatedAt time.Time + ContentDropped bool +} + +// EffectivePerformer is the id the performers/exclude_performers filters +// match: the delegating agent when the action was delegated, else the creator. +func (r Record) EffectivePerformer() int64 { + if r.PerformedByID != nil { + return *r.PerformedByID + } + return r.CreatorID +} + +// RecordSeen writes the pointer as seen if the id is new, and reports whether +// it was. +// +// This is the dedupe, and it is durable on purpose. The poll lane runs about +// thirty seconds behind the live lane, so the ordinary case is the same event +// arriving twice; a restart in between would let an in-memory set answer "new" +// to the second copy and dispatch it a second time. +// +// A known id is never rewritten. That is what makes a terminal record a +// tombstone: the ledger keeps (id, state, timestamps) indefinitely even after +// its pointer payload is dropped, so an explicit replay of old history can +// never turn a finished event back into a new task. +// +// It also resolves the id against any loss, open or closed: an event the repair walk — or +// the ordinary poll lane, later — serves is one the overflow did not cost us. +func (l *Ledger) RecordSeen(ctx context.Context, ev eventfeed.Event, lane Lane) (bool, error) { + tx, err := l.db.BeginTx(ctx, nil) + if err != nil { + return false, fmt.Errorf("connector: begin record seen: %w", err) + } + defer func() { _ = tx.Rollback() }() + + now := l.timestamp() + res, err := tx.ExecContext(ctx, ` +INSERT INTO events ( + id, state, lane, event_type, kind, action, bucket_id, creator_id, + performed_by_id, recording_id, details, actor_type, visible_to_clients, + created_at, seen_at, updated_at +) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) +ON CONFLICT (id) DO NOTHING`, + ev.ID, string(StateSeen), string(lane), ev.EventType, ev.Kind, ev.Action, + ev.BucketID, ev.CreatorID, ev.PerformedByID, ev.RecordingID, + detailsArg(ev.Details), ev.ActorType, ev.VisibleToClients, + stamp(ev.CreatedAt), now, now) + if err != nil { + return false, fmt.Errorf("connector: record seen %d: %w", ev.ID, err) + } + affected, err := res.RowsAffected() + if err != nil { + return false, fmt.Errorf("connector: record seen %d: %w", ev.ID, err) + } + + // Missing or already given up on: an id intake has received is recovered, + // however late it came, and status must stop reporting it. + if _, err := tx.ExecContext(ctx, + `UPDATE loss_ids SET state = ? WHERE event_id = ? AND state IN (?, ?)`, + string(LossRecovered), ev.ID, string(LossMissing), string(LossUnrecovered)); err != nil { + return false, fmt.Errorf("connector: resolve loss for %d: %w", ev.ID, err) + } + + if err := tx.Commit(); err != nil { + return false, fmt.Errorf("connector: commit record seen %d: %w", ev.ID, err) + } + return affected == 1, nil +} + +// detailsArg keeps a details object out of the row when the event publishes +// none. Only boost.created and card.moved do; storing an empty blob for every +// other type would make "has details" unanswerable in SQL. +func detailsArg(details json.RawMessage) any { + if len(details) == 0 { + return nil + } + return []byte(details) +} + +// Get returns one record by event id. +func (l *Ledger) Get(ctx context.Context, id int64) (Record, bool, error) { + rows, err := l.db.QueryContext(ctx, selectRecords+` WHERE id = ?`, id) + if err != nil { + return Record{}, false, fmt.Errorf("connector: get event %d: %w", id, err) + } + records, err := scanRecords(rows) + if err != nil || len(records) == 0 { + return Record{}, false, err + } + return records[0], true, nil +} + +// RecordsInStateAfter returns up to limit records in state whose id is above +// afterID, oldest first — the paging form of RecordsInState. +func (l *Ledger) RecordsInStateAfter(ctx context.Context, state RecordState, afterID int64, limit int) ([]Record, error) { + rows, err := l.db.QueryContext(ctx, + selectRecords+` WHERE state = ? AND id > ? ORDER BY id LIMIT ?`, string(state), afterID, limit) + if err != nil { + return nil, fmt.Errorf("connector: list %s records: %w", state, err) + } + return scanRecords(rows) +} + +// RecordsInState returns up to limit records in state, oldest event first. +// Every non-terminal state is re-run on start, and this is how they are found. +func (l *Ledger) RecordsInState(ctx context.Context, state RecordState, limit int) ([]Record, error) { + rows, err := l.db.QueryContext(ctx, selectRecords+` WHERE state = ? ORDER BY id LIMIT ?`, string(state), limit) + if err != nil { + return nil, fmt.Errorf("connector: list %s records: %w", state, err) + } + return scanRecords(rows) +} + +// CountInState counts the records in state. +func (l *Ledger) CountInState(ctx context.Context, state RecordState) (int, error) { + var n int + err := l.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM events WHERE state = ?`, string(state)).Scan(&n) + if err != nil { + return 0, fmt.Errorf("connector: count %s records: %w", state, err) + } + return n, nil +} + +// lifecycle is the ledger's state machine: for each state, the states a record +// may move to from it. +// +// Intake writes only seen; the rest is written by admission and dispatch. The +// table lives here anyway, because the guarantee it makes is the ledger's: a +// completed or discarded record is finished, and a store that lets one move +// back into the working states is a store where an event can be dispatched +// twice, or where a tombstone whose payload has been dropped is picked up as +// work with nothing in it. +// +// Blocked is not terminal on purpose: it is retained and retried, so it has +// edges back into the working states. Completed and discarded have none. +var lifecycle = map[RecordState][]RecordState{ + // seen to queued is one edge, not two: admission commits an admitted + // verdict AS queued when the conversation is already live, so the record + // never passes through admitted at all. + StateSeen: {StateAdmitted, StateQueued, StateBlocked, StateDiscarded}, + StateAdmitted: {StateQueued, StateDispatched, StateBlocked, StateDiscarded}, + StateQueued: {StateDispatched, StateBlocked, StateDiscarded}, + StateBlocked: {StateAdmitted, StateQueued, StateDispatched, StateDiscarded}, + // A dispatched record whose worker never started has its exposure + // withdrawn and returns to admitted. It is never discarded: a dispatched + // event ends completed, with an outcome, even when the outcome is + // unknown. + StateDispatched: {StateCompleted, StateBlocked, StateAdmitted}, + StateCompleted: nil, + StateDiscarded: nil, +} + +// enterableFrom is the states a record may be in for a move to target to be +// allowed, target itself included: writing the state a record already has is a +// repeat, not a move, so a retry after a crash is not an error. +func enterableFrom(target RecordState) []string { + froms := []string{string(target)} + for from, tos := range lifecycle { + if slices.Contains(tos, target) { + froms = append(froms, string(from)) + } + } + // Sorted so the statement is the same every time it is built, whatever + // order the map ranges in. + slices.Sort(froms) + return froms +} + +// ErrNotATransition reports a state change the lifecycle does not have. +var ErrNotATransition = errors.New("not a transition the ledger's lifecycle allows") + +// SetState moves a record to state with a reason, which must be empty for +// every state but blocked and discarded — those two are the only ones a reason +// explains. +// +// The move is refused unless the lifecycle has that edge, and it is refused by +// the UPDATE itself rather than by a read before it: a check and a write in +// two statements is a race, and this is the guarantee that a finished record +// stays finished. +func (l *Ledger) SetState(ctx context.Context, id int64, state RecordState, reason string) error { + switch state { + case StateBlocked, StateDiscarded: + if reason == "" { + return fmt.Errorf("connector: set state of %d: a %s record needs a reason", id, state) + } + case StateSeen, StateAdmitted, StateQueued, StateDispatched, StateCompleted: + if reason != "" { + return fmt.Errorf("connector: set state of %d: a %s record takes no reason", id, state) + } + default: + // A state outside the lifecycle is a row no recovery scan looks for. + return fmt.Errorf("connector: set state of %d: %q is not a ledger state", id, state) + } + froms := enterableFrom(state) + args := []any{string(state), reason, string(state), l.timestamp(), id} + for _, from := range froms { + args = append(args, from) + } + // The only thing concatenated is a list of "?" as long as the lifecycle's + // own edge list. Every value is bound. + //nolint:gosec // G202: placeholders, not values + // + // updated_at is left alone when the state does not change. It is the + // retention clock DropContent reads, and a repeated write of the state a + // record already has would silently restart the window on a finished + // record. + query := `UPDATE events SET state = ?, reason = ?, + updated_at = CASE WHEN state = ? THEN updated_at ELSE ? END +WHERE id = ? AND state IN (` + strings.TrimSuffix(strings.Repeat("?, ", len(froms)), ", ") + `)` + res, err := l.db.ExecContext(ctx, query, args...) + if err != nil { + return fmt.Errorf("connector: set state of %d: %w", id, err) + } + affected, err := res.RowsAffected() + if err != nil { + return fmt.Errorf("connector: set state of %d: %w", id, err) + } + if affected == 0 { + return l.explainRefusal(ctx, id, state) + } + return nil +} + +// explainRefusal says why an update changed nothing: there is no such record, +// or the record is somewhere the lifecycle cannot leave for state. +// +// The refusal itself already happened, in the UPDATE. This is the message, and +// it reads the row a second time: under a concurrent writer it can name a +// state the record has since left. Diagnostics, not a verdict to act on. +func (l *Ledger) explainRefusal(ctx context.Context, id int64, state RecordState) error { + var current string + switch err := l.db.QueryRowContext(ctx, `SELECT state FROM events WHERE id = ?`, id).Scan(¤t); { + case errors.Is(err, sql.ErrNoRows): + return fmt.Errorf("connector: set state of %d: %w", id, ErrNoSuchRecord) + case err != nil: + return fmt.Errorf("connector: set state of %d: %w", id, err) + } + return fmt.Errorf("connector: set state of %d: %s to %s is %w", id, current, state, ErrNotATransition) +} + +// ErrNoSuchRecord reports a state change addressed at an id the ledger does +// not hold. +var ErrNoSuchRecord = errors.New("no such event record") + +// DropContent drops the pointer payload of terminal records past their +// retention window and leaves the tombstone: id, state, outcome timestamps. +// +// The tombstone is what makes an explicit replay safe forever, so it is never +// deleted — only the payload goes. Non-terminal records are never touched: +// their payload is the only copy of what intake was told. +func (l *Ledger) DropContent(ctx context.Context, discardedBefore, completedBefore time.Time) (int, error) { + res, err := l.db.ExecContext(ctx, ` +UPDATE events +SET details = NULL, event_type = '', kind = '', action = '', bucket_id = 0, + creator_id = 0, performed_by_id = NULL, recording_id = 0, actor_type = '', + visible_to_clients = NULL, content_dropped = 1, updated_at = updated_at +WHERE content_dropped = 0 + AND ((state = ? AND updated_at < ?) OR (state = ? AND updated_at < ?))`, + string(StateDiscarded), stamp(discardedBefore), + string(StateCompleted), stamp(completedBefore)) + if err != nil { + return 0, fmt.Errorf("connector: drop content: %w", err) + } + affected, err := res.RowsAffected() + if err != nil { + return 0, fmt.Errorf("connector: drop content: %w", err) + } + return int(affected), nil +} + +const selectRecords = ` +SELECT id, state, reason, lane, event_type, kind, action, bucket_id, creator_id, + performed_by_id, recording_id, details, actor_type, visible_to_clients, + created_at, seen_at, updated_at, content_dropped +FROM events` + +func scanRecords(rows *sql.Rows) ([]Record, error) { + defer func() { _ = rows.Close() }() + + var records []Record + for rows.Next() { + var ( + r Record + state, lane string + details []byte + createdAt, seenAt, updatedAt string + contentDropped int + performedBy sql.NullInt64 + visibleToClients sql.NullBool + ) + if err := rows.Scan(&r.ID, &state, &r.Reason, &lane, &r.EventType, &r.Kind, + &r.Action, &r.BucketID, &r.CreatorID, &performedBy, &r.RecordingID, + &details, &r.ActorType, &visibleToClients, &createdAt, &seenAt, + &updatedAt, &contentDropped); err != nil { + return nil, fmt.Errorf("connector: scan event record: %w", err) + } + r.State = RecordState(state) + r.Lane = Lane(lane) + if performedBy.Valid { + id := performedBy.Int64 + r.PerformedByID = &id + } + if visibleToClients.Valid { + v := visibleToClients.Bool + r.VisibleToClients = &v + } + if len(details) > 0 { + r.Details = json.RawMessage(details) + } + var err error + if r.CreatedAt, err = parseStamp(createdAt); err != nil { + return nil, err + } + if r.SeenAt, err = parseStamp(seenAt); err != nil { + return nil, err + } + if r.UpdatedAt, err = parseStamp(updatedAt); err != nil { + return nil, err + } + r.ContentDropped = contentDropped != 0 + records = append(records, r) + } + return records, rows.Err() +} + +func parseStamp(s string) (time.Time, error) { + t, err := time.Parse(time.RFC3339Nano, s) + if err != nil { + return time.Time{}, fmt.Errorf("connector: parse ledger timestamp %q: %w", s, err) + } + return t, nil +} diff --git a/internal/connector/ledger_recovery.go b/internal/connector/ledger_recovery.go new file mode 100644 index 000000000..8e82192fd --- /dev/null +++ b/internal/connector/ledger_recovery.go @@ -0,0 +1,437 @@ +package connector + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" +) + +// LossState is what became of one id the live buffer dropped. +type LossState string + +const ( + // LossMissing is an id the overflow dropped and nothing has served since. + LossMissing LossState = "missing" + // LossNeverLost is an id the ledger already held when the overflow was + // signaled. It was dropped from the buffer, not from the connector. + LossNeverLost LossState = "never_lost" + // LossRecovered is an id a later poll — the repair walk or the ordinary + // lane — served. + LossRecovered LossState = "recovered" + // LossUnrecovered is an id still missing when the repair window closed: a + // late straggler, a recording deleted before it became poll-visible, or + // history behind the epoch. It is reported, never hidden. + LossUnrecovered LossState = "unrecovered" +) + +// Loss is one accepted buffer overflow and the state of its reconciliation. +type Loss struct { + ID int64 + DetectedAt time.Time + DroppedCount int + // RepairSince is the walk's entry: one below the lowest missing id. + RepairSince int64 + // RepairCursor is the walk's own position, kept here and never written to + // the feed's checkpoint. + RepairCursor string + DeadlineAt time.Time + ResolvedAt *time.Time + // Filters is the filter set this loss was recorded under. The repair walk + // uses it, whatever the connector's current filters are: a loss recorded + // under one set and walked under another is repaired against a lane that + // never carried its events. + Filters eventfeed.Filters + // HasFilters distinguishes a loss recorded with no filters — the + // whole-account feed, the common case — from one written before losses + // carried them at all. The first is walked with no filters; the second + // with the connector's own, which is the best that row can say. + HasFilters bool +} + +// GapClass distinguishes the feed's two 410s. They mean different things and +// must never share a recovery path. +type GapClass string + +const ( + // GapEpoch is the account feed's 410: the held position fell below the + // feed's epoch, and epoch_after_id names where servable history begins. + // The served resume URL is followed as given. + GapEpoch GapClass = "epoch" + // GapRetention is the inbox lane's 410: the held position fell out of the + // 30-day retention window, and there is no epoch — epoch_after_id is + // absent, not zero. The account lane must never produce one, and a + // connector that sees one has been told something it does not model. + GapRetention GapClass = "retention" +) + +// EntryClass is how the feed re-entered after a 410, decided by the cursor the +// server served — never by what the connector would have chosen. +type EntryClass string + +const ( + // EntryPresent is a resume at since=now: the entry takes the head and + // history below it is gone. + EntryPresent EntryClass = "present" + // EntryReplay is a resume at since=: the entry is a replay that + // checkpoints from its first poll page. + EntryReplay EntryClass = "replay" + // EntryUnknown is a resume whose cursor could not be read. + EntryUnknown EntryClass = "unknown" +) + +// Gap is one recorded 410. +type Gap struct { + ID int64 + DetectedAt time.Time + Class GapClass + // EpochAfterID is set exactly when Class is GapEpoch. It is a pointer + // because the distinction between "absent" and "zero" is the whole of the + // difference between the two 410s. + EpochAfterID *int64 + EntryClass EntryClass + Note string +} + +// RecordLoss writes an accepted buffer overflow, in one transaction, before +// the handler returns Accept. +// +// Accept means the consumer owns the incompleteness. Owning it starts with it +// being on disk: a crash a millisecond later must still find the loss and +// resume its reconciliation, and a loss that only ever lived in memory is one +// the next start would never know to repair. The corollary is that a failed +// write is a reason to refuse the signal, not to accept it anyway. +// +// Dropped ids the ledger already holds were lost from the buffer, not from the +// connector; they are recorded as never_lost so the repair walk does not go +// looking for events it already has. +func (l *Ledger) RecordLoss(ctx context.Context, droppedIDs []int64, now time.Time, window time.Duration, filters eventfeed.Filters) (Loss, error) { + if len(droppedIDs) == 0 { + return Loss{}, errors.New("connector: buffer overflow with no dropped ids") + } + + tx, err := l.db.BeginTx(ctx, nil) + if err != nil { + return Loss{}, fmt.Errorf("connector: begin record loss: %w", err) + } + defer func() { _ = tx.Rollback() }() + + states := make(map[int64]LossState, len(droppedIDs)) + lowestMissing := int64(0) + for _, id := range droppedIDs { + if _, seen := states[id]; seen { + continue + } + var one int + err := tx.QueryRowContext(ctx, `SELECT 1 FROM events WHERE id = ?`, id).Scan(&one) + switch { + case errors.Is(err, sql.ErrNoRows): + states[id] = LossMissing + if lowestMissing == 0 || id < lowestMissing { + lowestMissing = id + } + case err != nil: + return Loss{}, fmt.Errorf("connector: classify dropped id %d: %w", id, err) + default: + states[id] = LossNeverLost + } + } + + encodedFilters, err := json.Marshal(filters) + if err != nil { + return Loss{}, fmt.Errorf("connector: encode loss filters: %w", err) + } + + loss := Loss{ + Filters: filters, + HasFilters: true, + DetectedAt: now.UTC(), + DroppedCount: len(states), + // One below the lowest missing id, so the walk's first page can serve + // it: the feed's since is exclusive. + RepairSince: lowestMissing - 1, + DeadlineAt: now.UTC().Add(window), + } + if lowestMissing == 0 { + // Every dropped id was already in the ledger. There is nothing to + // walk for, but the overflow still happened and is still recorded. + loss.RepairSince = 0 + resolved := now.UTC() + loss.ResolvedAt = &resolved + } + + res, err := tx.ExecContext(ctx, + `INSERT INTO losses (detected_at, dropped_count, repair_since, deadline_at, resolved_at, filters) VALUES (?, ?, ?, ?, ?, ?)`, + stamp(loss.DetectedAt), loss.DroppedCount, loss.RepairSince, stamp(loss.DeadlineAt), nullableStamp(loss.ResolvedAt), string(encodedFilters)) + if err != nil { + return Loss{}, fmt.Errorf("connector: insert loss: %w", err) + } + if loss.ID, err = res.LastInsertId(); err != nil { + return Loss{}, fmt.Errorf("connector: insert loss: %w", err) + } + + for id, state := range states { + if _, err := tx.ExecContext(ctx, + `INSERT INTO loss_ids (loss_id, event_id, state) VALUES (?, ?, ?)`, + loss.ID, id, string(state)); err != nil { + return Loss{}, fmt.Errorf("connector: insert dropped id %d: %w", id, err) + } + } + + if err := tx.Commit(); err != nil { + return Loss{}, fmt.Errorf("connector: commit loss: %w", err) + } + return loss, nil +} + +// OpenLosses returns the losses whose reconciliation has not finished, oldest +// first. Reconciliation resumes from this on every start. +func (l *Ledger) OpenLosses(ctx context.Context) ([]Loss, error) { + rows, err := l.db.QueryContext(ctx, ` +SELECT id, detected_at, dropped_count, repair_since, repair_cursor, deadline_at, resolved_at, filters +FROM losses WHERE resolved_at IS NULL ORDER BY id`) + if err != nil { + return nil, fmt.Errorf("connector: list open losses: %w", err) + } + defer func() { _ = rows.Close() }() + + var losses []Loss + for rows.Next() { + var ( + loss Loss + detected, deadline string + resolved sql.NullString + ) + var encodedFilters string + if err := rows.Scan(&loss.ID, &detected, &loss.DroppedCount, &loss.RepairSince, + &loss.RepairCursor, &deadline, &resolved, &encodedFilters); err != nil { + return nil, fmt.Errorf("connector: scan loss: %w", err) + } + if encodedFilters != "" { + loss.HasFilters = true + if err := json.Unmarshal([]byte(encodedFilters), &loss.Filters); err != nil { + return nil, fmt.Errorf("connector: decode loss filters: %w", err) + } + } + var err error + if loss.DetectedAt, err = parseStamp(detected); err != nil { + return nil, err + } + if loss.DeadlineAt, err = parseStamp(deadline); err != nil { + return nil, err + } + if resolved.Valid { + t, err := parseStamp(resolved.String) + if err != nil { + return nil, err + } + loss.ResolvedAt = &t + } + losses = append(losses, loss) + } + return losses, rows.Err() +} + +// MissingIDs returns the ids of a loss still in state, ascending. +func (l *Ledger) MissingIDs(ctx context.Context, lossID int64, state LossState) ([]int64, error) { + rows, err := l.db.QueryContext(ctx, + `SELECT event_id FROM loss_ids WHERE loss_id = ? AND state = ? ORDER BY event_id`, + lossID, string(state)) + if err != nil { + return nil, fmt.Errorf("connector: list loss ids: %w", err) + } + defer func() { _ = rows.Close() }() + + var ids []int64 + for rows.Next() { + var id int64 + if err := rows.Scan(&id); err != nil { + return nil, fmt.Errorf("connector: scan loss id: %w", err) + } + ids = append(ids, id) + } + return ids, rows.Err() +} + +// ExtendLossDeadline pushes a loss's window out, for time the walk spent +// waiting on the server rather than working. +func (l *Ledger) ExtendLossDeadline(ctx context.Context, lossID int64, deadline time.Time) error { + _, err := l.db.ExecContext(ctx, + `UPDATE losses SET deadline_at = ? WHERE id = ?`, stamp(deadline.UTC()), lossID) + if err != nil { + return fmt.Errorf("connector: extend loss deadline: %w", err) + } + return nil +} + +// SaveRepairCursor records where the repair walk has reached. +// +// This is the walk's own cursor and it is stored on the loss, never on the +// feed's checkpoint. The walk is seeded from a live id that ran far ahead of +// the poll lane; a checkpoint taken from it would skip every event still +// inside the safety delay behind it. +func (l *Ledger) SaveRepairCursor(ctx context.Context, lossID int64, cursor string) error { + _, err := l.db.ExecContext(ctx, + `UPDATE losses SET repair_cursor = ? WHERE id = ?`, cursor, lossID) + if err != nil { + return fmt.Errorf("connector: save repair cursor: %w", err) + } + return nil +} + +// CloseLoss ends a loss's reconciliation: everything still missing becomes +// unrecovered, and the loss is resolved. It reports how many ids were left +// unrecovered. +func (l *Ledger) CloseLoss(ctx context.Context, lossID int64, now time.Time) (int, error) { + tx, err := l.db.BeginTx(ctx, nil) + if err != nil { + return 0, fmt.Errorf("connector: begin close loss: %w", err) + } + defer func() { _ = tx.Rollback() }() + + res, err := tx.ExecContext(ctx, + `UPDATE loss_ids SET state = ? WHERE loss_id = ? AND state = ?`, + string(LossUnrecovered), lossID, string(LossMissing)) + if err != nil { + return 0, fmt.Errorf("connector: mark unrecovered: %w", err) + } + unrecovered, err := res.RowsAffected() + if err != nil { + return 0, fmt.Errorf("connector: mark unrecovered: %w", err) + } + + if _, err := tx.ExecContext(ctx, + `UPDATE losses SET resolved_at = ? WHERE id = ? AND resolved_at IS NULL`, + stamp(now.UTC()), lossID); err != nil { + return 0, fmt.Errorf("connector: resolve loss: %w", err) + } + + if err := tx.Commit(); err != nil { + return 0, fmt.Errorf("connector: commit close loss: %w", err) + } + return int(unrecovered), nil +} + +// SetRepairEntry moves a loss's walk entry to since and drops its cursor — +// after a 410 has fenced off everything at or below since. +func (l *Ledger) SetRepairEntry(ctx context.Context, lossID, since int64) error { + _, err := l.db.ExecContext(ctx, + `UPDATE losses SET repair_since = ?, repair_cursor = '' WHERE id = ?`, since, lossID) + if err != nil { + return fmt.Errorf("connector: set repair entry: %w", err) + } + return nil +} + +// MarkUnrecoveredThrough gives up on a loss's missing ids at or below id — the +// ones a 410's epoch has fenced off — and leaves the rest missing. +func (l *Ledger) MarkUnrecoveredThrough(ctx context.Context, lossID, id int64) (int, error) { + res, err := l.db.ExecContext(ctx, + `UPDATE loss_ids SET state = ? WHERE loss_id = ? AND state = ? AND event_id <= ?`, + string(LossUnrecovered), lossID, string(LossMissing), id) + if err != nil { + return 0, fmt.Errorf("connector: mark unrecovered below the epoch: %w", err) + } + n, err := res.RowsAffected() + if err != nil { + return 0, fmt.Errorf("connector: mark unrecovered below the epoch: %w", err) + } + return int(n), nil +} + +// UnrecoveredIDs returns every id the connector has given up on, across all +// losses. `basecamp connect status` and `doctor` show these: an unrecovered id +// is documented, not hidden, and the poll lane may still serve it later. +func (l *Ledger) UnrecoveredIDs(ctx context.Context) ([]int64, error) { + rows, err := l.db.QueryContext(ctx, + `SELECT DISTINCT event_id FROM loss_ids WHERE state = ? ORDER BY event_id`, + string(LossUnrecovered)) + if err != nil { + return nil, fmt.Errorf("connector: list unrecovered ids: %w", err) + } + defer func() { _ = rows.Close() }() + + var ids []int64 + for rows.Next() { + var id int64 + if err := rows.Scan(&id); err != nil { + return nil, fmt.Errorf("connector: scan unrecovered id: %w", err) + } + ids = append(ids, id) + } + return ids, rows.Err() +} + +// RecordGap writes one 410. Nothing is posted to Basecamp for a gap; it is a +// fact about the feed, shown by status and doctor. +func (l *Ledger) RecordGap(ctx context.Context, gap Gap) (int64, error) { + switch gap.Class { + case GapEpoch: + if gap.EpochAfterID == nil { + return 0, errors.New("connector: an epoch gap must carry epoch_after_id") + } + case GapRetention: + if gap.EpochAfterID != nil { + return 0, errors.New("connector: a retention gap has no epoch_after_id") + } + default: + return 0, fmt.Errorf("connector: unknown gap class %q", gap.Class) + } + + res, err := l.db.ExecContext(ctx, + `INSERT INTO gaps (detected_at, class, epoch_after_id, entry_class, note) VALUES (?, ?, ?, ?, ?)`, + stamp(gap.DetectedAt.UTC()), string(gap.Class), gap.EpochAfterID, string(gap.EntryClass), gap.Note) + if err != nil { + return 0, fmt.Errorf("connector: record gap: %w", err) + } + return res.LastInsertId() +} + +// Gaps returns every recorded 410, oldest first. +func (l *Ledger) Gaps(ctx context.Context) ([]Gap, error) { + rows, err := l.db.QueryContext(ctx, + `SELECT id, detected_at, class, epoch_after_id, entry_class, note FROM gaps ORDER BY id`) + if err != nil { + return nil, fmt.Errorf("connector: list gaps: %w", err) + } + defer func() { _ = rows.Close() }() + + var gaps []Gap + for rows.Next() { + var ( + gap Gap + detected string + class string + entryClass string + epoch sql.NullInt64 + ) + if err := rows.Scan(&gap.ID, &detected, &class, &epoch, &entryClass, &gap.Note); err != nil { + return nil, fmt.Errorf("connector: scan gap: %w", err) + } + var err error + if gap.DetectedAt, err = parseStamp(detected); err != nil { + return nil, err + } + gap.Class = GapClass(class) + gap.EntryClass = EntryClass(entryClass) + if epoch.Valid { + id := epoch.Int64 + gap.EpochAfterID = &id + } + gaps = append(gaps, gap) + } + return gaps, rows.Err() +} + +func stamp(t time.Time) string { return t.UTC().Format(ledgerTime) } + +func nullableStamp(t *time.Time) any { + if t == nil { + return nil + } + return stamp(*t) +} diff --git a/internal/connector/ledger_test.go b/internal/connector/ledger_test.go new file mode 100644 index 000000000..0c94ceb8f --- /dev/null +++ b/internal/connector/ledger_test.go @@ -0,0 +1,240 @@ +package connector + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" + + "github.com/basecamp/basecamp-cli/internal/connector/setup" +) + +func newTestLedger(t *testing.T) *Ledger { + t.Helper() + ledger, err := OpenLedger(filepath.Join(t.TempDir(), "state", "connector.db")) + require.NoError(t, err) + t.Cleanup(func() { _ = ledger.Close() }) + return ledger +} + +func testEvent(id int64) eventfeed.Event { + return eventfeed.Event{ + ID: id, + Kind: "comment_created", + EventType: "comment.created", + Action: "created", + CreatedAt: time.Date(2026, 9, 16, 10, 0, 0, 0, time.UTC), + BucketID: 48699913, + CreatorID: 26909558, + RecordingID: 10304028972, + } +} + +func testKey() eventfeed.CheckpointKey { + return eventfeed.CheckpointKey{ + Origin: "https://3.basecampapi.com", + AccountID: "2914079", + ConsumerNamespace: "connector-test", + FilterKey: "srv2-9f2ab04e5c11d3a7", + } +} + +func TestRecordSeenDedupesAcrossLanes(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + + fresh, err := ledger.RecordSeen(ctx, testEvent(17099838500), LaneLive) + require.NoError(t, err) + assert.True(t, fresh) + + // The poll lane serving what the live lane already delivered is the + // ordinary case, not an anomaly. + fresh, err = ledger.RecordSeen(ctx, testEvent(17099838500), LanePoll) + require.NoError(t, err) + assert.False(t, fresh) + + record, ok, err := ledger.Get(ctx, 17099838500) + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, LaneLive, record.Lane, "the first lane to serve it stands") + assert.Equal(t, StateSeen, record.State) +} + +func TestRecordSeenKeepsDetailsVerbatimAndNilWhenAbsent(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + + withDetails := testEvent(1) + withDetails.EventType = "card.moved" + withDetails.Details = json.RawMessage(`{"column_id":10287321848,"previous_column_id":null}`) + _, err := ledger.RecordSeen(ctx, withDetails, LanePoll) + require.NoError(t, err) + + _, err = ledger.RecordSeen(ctx, testEvent(2), LanePoll) + require.NoError(t, err) + + moved, _, err := ledger.Get(ctx, 1) + require.NoError(t, err) + assert.JSONEq(t, `{"column_id":10287321848,"previous_column_id":null}`, string(moved.Details)) + + plain, _, err := ledger.Get(ctx, 2) + require.NoError(t, err) + assert.Nil(t, plain.Details, "an event that publishes no details must not be stored with an empty one") +} + +func TestDroppedContentLeavesATombstoneThatStillDedupes(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + + _, err := ledger.RecordSeen(ctx, testEvent(42), LanePoll) + require.NoError(t, err) + require.NoError(t, ledger.SetState(ctx, 42, StateDiscarded, "untrusted_author")) + + dropped, err := ledger.DropContent(ctx, time.Now().Add(time.Hour), time.Now().Add(time.Hour)) + require.NoError(t, err) + assert.Equal(t, 1, dropped) + + record, ok, err := ledger.Get(ctx, 42) + require.NoError(t, err) + require.True(t, ok) + assert.True(t, record.ContentDropped) + assert.Empty(t, record.EventType, "the payload goes") + assert.Equal(t, StateDiscarded, record.State, "the outcome stays") + + // The whole point of keeping the tombstone: an explicit replay of old + // history can never turn a finished event back into a new task. + fresh, err := ledger.RecordSeen(ctx, testEvent(42), LanePoll) + require.NoError(t, err) + assert.False(t, fresh) +} + +func TestDropContentLeavesNonTerminalRecordsAlone(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + + _, err := ledger.RecordSeen(ctx, testEvent(7), LanePoll) + require.NoError(t, err) + require.NoError(t, ledger.SetState(ctx, 7, StateBlocked, "read_failed")) + + dropped, err := ledger.DropContent(ctx, time.Now().Add(time.Hour), time.Now().Add(time.Hour)) + require.NoError(t, err) + assert.Zero(t, dropped) + + record, _, err := ledger.Get(ctx, 7) + require.NoError(t, err) + assert.Equal(t, "comment.created", record.EventType, + "a blocked record's pointer is the only copy of what intake was told") +} + +func TestCheckpointRoundTrip(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + key := testKey() + + _, ok, err := ledger.Load(ctx, key) + require.NoError(t, err) + assert.False(t, ok) + + require.NoError(t, ledger.Save(ctx, key, "opaque-position-1")) + position, ok, err := ledger.Load(ctx, key) + require.NoError(t, err) + assert.True(t, ok) + assert.Equal(t, "opaque-position-1", position) +} + +func TestForgetPositionKeepsTheLastPollServedID(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + key := testKey() + + require.NoError(t, ledger.Save(ctx, key, "opaque-position-1")) + require.NoError(t, ledger.NotePollServed(ctx, key, 17099838600)) + require.NoError(t, ledger.ForgetPosition(ctx, key)) + + _, ok, err := ledger.Load(ctx, key) + require.NoError(t, err) + assert.False(t, ok, "a dropped position must not read back as a present empty one") + + served, err := ledger.LastPollServedID(ctx, key) + require.NoError(t, err) + assert.Equal(t, int64(17099838600), served, + "the 409 invalidates the position, not the id the poll lane had reached") +} + +func TestNotePollServedOnlyMovesForward(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + key := testKey() + + require.NoError(t, ledger.NotePollServed(ctx, key, 500)) + require.NoError(t, ledger.NotePollServed(ctx, key, 400)) + + served, err := ledger.LastPollServedID(ctx, key) + require.NoError(t, err) + assert.Equal(t, int64(500), served) +} + +func TestCheckpointsAreKeyedByFilterDigest(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + + narrow := testKey() + wide := testKey() + wide.FilterKey = "srv2-0000000000000000" + + require.NoError(t, ledger.Save(ctx, narrow, "narrow-position")) + _, ok, err := ledger.Load(ctx, wide) + require.NoError(t, err) + assert.False(t, ok, "a filter change re-enters under its own lineage") +} + +// The ledger holds feed positions, which resume the account's feed. A path +// that can be redirected is a ledger that can be read, so what is validated +// is the file that gets opened, not the name that was asked for. +func TestLedgerRefusesASymlinkedPath(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.Chmod(dir, 0o700)) + elsewhere := filepath.Join(t.TempDir(), "elsewhere.db") + require.NoError(t, os.WriteFile(elsewhere, nil, 0o600)) + path := filepath.Join(dir, "connector.db") + require.NoError(t, os.Symlink(elsewhere, path)) + + _, err := OpenLedger(path) + + require.Error(t, err) + assert.ErrorIs(t, err, setup.ErrNotPrivate) +} + +func TestLedgerRefusesASymlinkedParent(t *testing.T) { + target := filepath.Join(t.TempDir(), "target") + require.NoError(t, os.Mkdir(target, 0o700)) + link := filepath.Join(t.TempDir(), "state") + require.NoError(t, os.Symlink(target, link)) + + _, err := OpenLedger(filepath.Join(link, "connector.db")) + + require.Error(t, err) + assert.ErrorIs(t, err, setup.ErrNotPrivate) +} + +// A 0700 directory under an ancestor anyone can write is not private: the +// ancestor's owner can rename it away and put their own in its place. +func TestLedgerRefusesALooseAncestor(t *testing.T) { + loose := filepath.Join(t.TempDir(), "loose") + require.NoError(t, os.Mkdir(loose, 0o700)) + require.NoError(t, os.Chmod(loose, 0o777)) // after the umask, not through it + dir := filepath.Join(loose, "state") + require.NoError(t, os.Mkdir(dir, 0o700)) + + _, err := OpenLedger(filepath.Join(dir, "connector.db")) + + require.Error(t, err) + assert.ErrorIs(t, err, setup.ErrNotPrivate) +} diff --git a/internal/connector/lock.go b/internal/connector/lock.go new file mode 100644 index 000000000..4a99c4119 --- /dev/null +++ b/internal/connector/lock.go @@ -0,0 +1,107 @@ +package connector + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strconv" + "time" + + "github.com/basecamp/basecamp-cli/internal/connector/setup" +) + +// ErrAlreadyRunning reports a second connector for the same agent. +var ErrAlreadyRunning = errors.New("connector: another connector already holds this account and agent") + +// InstanceLock is the refusal of a second connector on one agent identity. +// +// It is keyed on the ACCOUNT and the agent's Person id, not on the profile +// name. Two profiles can hold credentials for the same agent, and the failure +// this prevents is not "the same configuration twice" — it is one agent's +// mentions being dispatched twice, which is a property of the identity, not of +// the file that names it. +// +// The kernel drops a flock when the holding descriptor closes, process death +// included, so a crashed connector cannot wedge the lock and there is no +// stale-lock reaping to get wrong. The metadata written beside it is +// diagnostic only: the lock is the lock. +type InstanceLock struct { + unlock func() error + path string +} + +// instanceHolder is what a running connector writes beside its lock so the +// refusal can say who is holding it. +type instanceHolder struct { + PID int `json:"pid"` + StartedAt string `json:"started_at"` + AccountID string `json:"account_id"` + AgentID int64 `json:"agent_person_id"` +} + +// AcquireInstanceLock takes the lock for one account and agent, or refuses. +func AcquireInstanceLock(dir, accountID string, agentPersonID int64, now time.Time) (*InstanceLock, error) { + account, err := strconv.ParseUint(accountID, 10, 64) + if err != nil || account == 0 || agentPersonID <= 0 { + return nil, errors.New("connector: the instance lock needs a numeric account id and an agent person id") + } + // The file is named from the NUMBER, not the spelling: "02914079" and + // "2914079" are one account and must meet one lock, or the refusal of a + // second connector is a matter of how the id was typed. + accountID = strconv.FormatUint(account, 10) + + path := filepath.Join(dir, "instance-"+accountID+"-"+strconv.FormatInt(agentPersonID, 10)+".lock") + // The lock is only a lock if nobody else can reach it. A directory that + // merely EXISTS at 0700 is not enough — MkdirAll leaves a group-writable + // one exactly as it found it — so the directory, everything above it and + // the lock file itself are vetted here, by the same check the connector's + // trust file gets. Someone who can write the directory can point two + // connectors at two inodes, and then neither excludes the other: one + // agent's mentions dispatched twice, which is the failure this lock + // exists to prevent. + unlock, err := setup.TryLockPrivate(path) + switch { + case errors.Is(err, setup.ErrLockHeld): + return nil, fmt.Errorf("%w: %s", ErrAlreadyRunning, describeHolder(path)) + case err != nil: + // No lock means no connector. There is no degraded mode here. + return nil, fmt.Errorf("connector: take the instance lock: %w", err) + } + + holder, err := json.Marshal(instanceHolder{ + PID: os.Getpid(), + StartedAt: now.UTC().Format(time.RFC3339), + AccountID: accountID, + AgentID: agentPersonID, + }) + if err == nil { + // Best effort, and deliberately after the lock is held: a connector + // that cannot describe itself still must not be a second connector. + _ = os.WriteFile(path+".json", append(holder, '\n'), 0o600) + } + + return &InstanceLock{unlock: unlock, path: path}, nil +} + +// Release drops the lock. +func (l *InstanceLock) Release() error { + _ = os.Remove(l.path + ".json") + return l.unlock() +} + +// Path is the lock file, for diagnostics. +func (l *InstanceLock) Path() string { return l.path } + +func describeHolder(path string) string { + raw, err := os.ReadFile(path + ".json") + if err != nil { + return "held by another process" + } + var holder instanceHolder + if err := json.Unmarshal(raw, &holder); err != nil { + return "held by another process" + } + return fmt.Sprintf("held by pid %d since %s", holder.PID, holder.StartedAt) +} diff --git a/internal/connector/lock_test.go b/internal/connector/lock_test.go new file mode 100644 index 000000000..ff3852b93 --- /dev/null +++ b/internal/connector/lock_test.go @@ -0,0 +1,125 @@ +package connector + +import ( + "errors" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/basecamp/basecamp-cli/internal/connector/setup" +) + +// Two connectors on one agent identity dispatch every mention twice. The Ruby +// connector's own notes record that happening. +func TestSecondConnectorOnTheSameAgentIsRefused(t *testing.T) { + dir := t.TempDir() + now := time.Now() + + first, err := AcquireInstanceLock(dir, "2914079", 52007412, now) + require.NoError(t, err) + t.Cleanup(func() { _ = first.Release() }) + + _, err = AcquireInstanceLock(dir, "2914079", 52007412, now) + require.ErrorIs(t, err, ErrAlreadyRunning) + assert.Contains(t, err.Error(), "pid", "the refusal says who is holding it") +} + +// The key is the identity, not the profile that names it: two profiles can +// hold credentials for one agent, and it is the agent that gets double-served. +func TestTheLockIsKeyedOnAccountAndAgent(t *testing.T) { + dir := t.TempDir() + now := time.Now() + + first, err := AcquireInstanceLock(dir, "2914079", 52007412, now) + require.NoError(t, err) + t.Cleanup(func() { _ = first.Release() }) + + otherAgent, err := AcquireInstanceLock(dir, "2914079", 26909558, now) + require.NoError(t, err, "a different agent in the same account is a different connector") + t.Cleanup(func() { _ = otherAgent.Release() }) + + otherAccount, err := AcquireInstanceLock(dir, "5951425", 52007412, now) + require.NoError(t, err, "the same agent id in another account is a different connector") + t.Cleanup(func() { _ = otherAccount.Release() }) +} + +func TestReleasedLockCanBeRetaken(t *testing.T) { + dir := t.TempDir() + now := time.Now() + + first, err := AcquireInstanceLock(dir, "2914079", 52007412, now) + require.NoError(t, err) + require.NoError(t, first.Release()) + + second, err := AcquireInstanceLock(dir, "2914079", 52007412, now) + require.NoError(t, err) + require.NoError(t, second.Release()) +} + +func TestLockRefusesAnIncompleteIdentity(t *testing.T) { + dir := t.TempDir() + _, err := AcquireInstanceLock(dir, "", 52007412, time.Now()) + assert.Error(t, err) + _, err = AcquireInstanceLock(dir, "2914079", 0, time.Now()) + assert.Error(t, err) +} + +// A lock in a directory other local users can write is not a lock: they can +// replace the file between two connectors' opens, and each then holds an +// inode the other never sees. +func TestTheLockRefusesADirectoryOthersCanWrite(t *testing.T) { + dir := filepath.Join(t.TempDir(), "state") + require.NoError(t, os.Mkdir(dir, 0o700)) + require.NoError(t, os.Chmod(dir, 0o770)) // after the umask, not through it + + _, err := AcquireInstanceLock(dir, "2914079", 52007412, time.Now()) + + require.Error(t, err) + assert.ErrorIs(t, err, setup.ErrNotPrivate) + assert.NotErrorIs(t, err, ErrAlreadyRunning, "a directory this connector will not use is not a second connector") +} + +// A lock path that is a symlink locks whatever it points at, so two +// connectors pointed at two symlinks exclude nobody. +func TestTheLockRefusesASymlinkedLockPath(t *testing.T) { + dir := t.TempDir() + elsewhere := filepath.Join(t.TempDir(), "elsewhere.lock") + require.NoError(t, os.WriteFile(elsewhere, nil, 0o600)) + require.NoError(t, os.Symlink(elsewhere, filepath.Join(dir, "instance-2914079-52007412.lock"))) + + _, err := AcquireInstanceLock(dir, "2914079", 52007412, time.Now()) + + require.Error(t, err) + assert.ErrorIs(t, err, setup.ErrNotPrivate) +} + +// The directory is created when it is missing, and created owner-only. +func TestTheLockCreatesItsDirectoryOwnerOnly(t *testing.T) { + dir := filepath.Join(t.TempDir(), "state") + + lock, err := AcquireInstanceLock(dir, "2914079", 52007412, time.Now()) + require.NoError(t, err) + t.Cleanup(func() { _ = lock.Release() }) + + info, err := os.Stat(dir) + require.NoError(t, err) + assert.Equal(t, os.FileMode(0o700), info.Mode().Perm()) + file, err := os.Stat(lock.Path()) + require.NoError(t, err) + assert.Zero(t, file.Mode().Perm()&0o077, "the lock file is the owner's alone") +} + +// A lock that cannot be taken is a refusal, never a connector running +// unlocked. +func TestTheLockRefusesWhenItCannotBeEstablished(t *testing.T) { + missing := filepath.Join(t.TempDir(), "absent", "state") + + _, err := AcquireInstanceLock(missing, "2914079", 52007412, time.Now()) + + require.Error(t, err) + assert.True(t, errors.Is(err, setup.ErrNotPrivate) || errors.Is(err, os.ErrNotExist), "got %v", err) +} diff --git a/internal/connector/loss_filters_test.go b/internal/connector/loss_filters_test.go new file mode 100644 index 000000000..c07eaa455 --- /dev/null +++ b/internal/connector/loss_filters_test.go @@ -0,0 +1,102 @@ +package connector + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" +) + +// C3: a loss is repaired under the filter set it was recorded with. A +// connector restarted with different filters would otherwise walk the wrong +// lane and condemn events the original filters would have served. +func TestALossIsRepairedUnderTheFilterSetItWasRecordedWith(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + recorded := eventfeed.Filters{Types: []string{"comment.created"}, Buckets: []int64{48699913}} + + loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, time.Now(), 10*time.Minute, recorded) + require.NoError(t, err) + assert.Equal(t, recorded.Types, loss.Filters.Types) + + // A later start under a different filter set. + open, err := ledger.OpenLosses(ctx) + require.NoError(t, err) + require.Len(t, open, 1) + require.Equal(t, recorded.Types, open[0].Filters.Types) + require.Equal(t, recorded.Buckets, open[0].Filters.Buckets) + + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + polls := &scriptedPolls{pages: []eventfeed.PollPage{ + {Events: []eventfeed.Event{testEvent(17099838509)}, Position: "p"}, + }} + walker, _ := newTestWalker(t, ledger, polls, clock) + walker.filters = eventfeed.Filters{Types: []string{"card.created"}} // the connector's current set + require.NoError(t, walker.reconcileLoss(ctx, open[0])) + + require.NotEmpty(t, polls.filters) + assert.Equal(t, recorded.Types, polls.filters[0].Types, + "the walk uses the loss's own filters, not the connector's current ones") + + recoveredIDs, err := ledger.MissingIDs(ctx, loss.ID, LossRecovered) + require.NoError(t, err) + assert.Equal(t, []int64{17099838509}, recoveredIDs) +} + +// The whole-account feed records a loss with no filters at all, which is not +// the same as a loss from before filters were stored. Restarted with filters +// added, the repair must still walk the whole account. +func TestALossRecordedWithNoFiltersIsRepairedWithNoFilters(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + _, err := ledger.RecordLoss(ctx, []int64{17099838509}, time.Now(), 10*time.Minute, eventfeed.Filters{}) + require.NoError(t, err) + + open, err := ledger.OpenLosses(ctx) + require.NoError(t, err) + require.Len(t, open, 1) + + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + polls := &scriptedPolls{pages: []eventfeed.PollPage{ + {Events: []eventfeed.Event{testEvent(17099838509)}, Position: "p"}, + }} + walker, _ := newTestWalker(t, ledger, polls, clock) + walker.filters = eventfeed.Filters{Types: []string{"comment.created"}} // added since + require.NoError(t, walker.reconcileLoss(ctx, open[0])) + + require.NotEmpty(t, polls.filters) + assert.Empty(t, polls.filters[0].Types, + "the loss was recorded on the whole account; a narrower lane never carried its events") +} + +// A loss written before losses carried their filters has none recorded, and is +// walked under the connector's own. +func TestALegacyLossWalksUnderTheConnectorsFilters(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + _, err := ledger.RecordLoss(ctx, []int64{17099838509}, time.Now(), 10*time.Minute, eventfeed.Filters{}) + require.NoError(t, err) + // What migration 2 leaves on a row written before it. + _, err = ledger.db.ExecContext(ctx, `UPDATE losses SET filters = ''`) + require.NoError(t, err) + + open, err := ledger.OpenLosses(ctx) + require.NoError(t, err) + require.Len(t, open, 1) + assert.False(t, open[0].HasFilters) + + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + polls := &scriptedPolls{pages: []eventfeed.PollPage{ + {Events: []eventfeed.Event{testEvent(17099838509)}, Position: "p"}, + }} + walker, _ := newTestWalker(t, ledger, polls, clock) + walker.filters = eventfeed.Filters{Types: []string{"comment.created"}} + require.NoError(t, walker.reconcileLoss(ctx, open[0])) + + require.NotEmpty(t, polls.filters) + assert.Equal(t, []string{"comment.created"}, polls.filters[0].Types) +} diff --git a/internal/connector/loss_schedule_test.go b/internal/connector/loss_schedule_test.go new file mode 100644 index 000000000..9bc2322cb --- /dev/null +++ b/internal/connector/loss_schedule_test.go @@ -0,0 +1,190 @@ +package connector + +import ( + "bytes" + "context" + "log/slog" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" +) + +// C2: a loss is always somewhere — repaired, scheduled for another attempt, or +// closed with a reason. Waiting on a server's own delay is not spending the +// window, so a throttle can neither condemn the ids nor run the clock out. +func TestAThrottleDoesNotSpendTheLossWindow(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + // A window with a minute left, and a server asking for fifteen. + loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, clock.at.Add(-9*time.Minute), 10*time.Minute, eventfeed.Filters{}) + require.NoError(t, err) + before := loss.DeadlineAt + + // Two refusals, each longer than what is left of the window, then the + // page. Only a window that does not spend on the server's own delays + // still has attempts left by then. + throttle := &eventfeed.PollError{Kind: eventfeed.PollThrottled, RetryAfter: 15 * time.Minute} + polls := &scriptedPolls{ + errs: []error{throttle, throttle}, + pages: []eventfeed.PollPage{{}, {}, {Events: []eventfeed.Event{testEvent(17099838509)}, Position: "p"}}, + } + walker, _ := newTestWalker(t, ledger, polls, clock) + walker.sleep = func(_ context.Context, d time.Duration) error { + clock.at = clock.at.Add(d) + return nil + } + require.NoError(t, walker.reconcile(ctx, loss)) + + unrecovered, err := ledger.UnrecoveredIDs(ctx) + require.NoError(t, err) + assert.Empty(t, unrecovered, "the server refused the attempt; it says nothing about the ids") + recovered, err := ledger.MissingIDs(ctx, loss.ID, LossRecovered) + require.NoError(t, err) + assert.Equal(t, []int64{17099838509}, recovered, "the retry after the wait found it") + _ = before +} + +// C2: a loss the repair queue had no room for is swept back in, without +// waiting for a restart. +func TestALossSkippedByAFullQueueIsSweptBackIn(t *testing.T) { + polls := &countingPolls{} + intake, ledger, _ := newTestIntake(t, polls, nil) + intake.opts.RepairInterval = time.Hour + intake.repairQueueSize = 1 + intake.repairSweep = 20 * time.Millisecond + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + for i := range 6 { + _, err := ledger.RecordLoss(ctx, []int64{int64(2000 + i)}, intake.now().Add(-time.Hour), time.Minute, eventfeed.Filters{}) + require.NoError(t, err) + } + require.NoError(t, intake.resumeReconciliation(ctx)) + + require.Eventually(t, func() bool { + open, err := ledger.OpenLosses(context.Background()) + return err == nil && len(open) == 0 + }, 10*time.Second, 20*time.Millisecond, "every open loss is eventually attempted, queue or no queue") + + cancel() + intake.repairs.Wait() +} + +// The same rule inside the window: the deadline moves out the moment the +// server asks for a wait, not only on the last attempt. +func TestAThrottleInsideTheWindowPostponesItImmediately(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, clock.at, time.Hour, eventfeed.Filters{}) + require.NoError(t, err) + + polls := &scriptedPolls{ + errs: []error{&eventfeed.PollError{Kind: eventfeed.PollThrottled, RetryAfter: 15 * time.Minute}}, + pages: []eventfeed.PollPage{{}, {Events: []eventfeed.Event{testEvent(17099838509)}, Position: "p"}}, + } + walker, _ := newTestWalker(t, ledger, polls, clock) + var atWait time.Time + walker.sleep = func(sleepCtx context.Context, d time.Duration) error { + if atWait.IsZero() { + open, err := ledger.OpenLosses(sleepCtx) + require.NoError(t, err) + require.Len(t, open, 1) + atWait = open[0].DeadlineAt + } + clock.at = clock.at.Add(d) + return nil + } + require.NoError(t, walker.reconcile(ctx, loss)) + + require.False(t, atWait.IsZero(), "the walk waited") + assert.Equal(t, loss.DeadlineAt.Add(15*time.Minute).UTC(), atWait.UTC(), + "the window moved out by exactly what the server asked for") +} + +// One throttle governs one pass. A later pass with a healthy server waits the +// ordinary cadence and does not push the window out again. +func TestOneThrottleDoesNotGovernEveryLaterPass(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, clock.at, time.Hour, eventfeed.Filters{}) + require.NoError(t, err) + + polls := &scriptedPolls{errs: []error{&eventfeed.PollError{Kind: eventfeed.PollThrottled, RetryAfter: 15 * time.Minute}}} + walker, _ := newTestWalker(t, ledger, polls, clock) + var waits []time.Duration + walker.sleep = func(sleepCtx context.Context, d time.Duration) error { + waits = append(waits, d) + clock.at = clock.at.Add(d) + if len(waits) >= 3 { + return context.Canceled + } + return nil + } + require.ErrorIs(t, walker.reconcile(ctx, loss), context.Canceled) + + require.Len(t, waits, 3) + assert.Equal(t, 15*time.Minute, waits[0], "the server asked for this one") + assert.Equal(t, walker.interval, waits[1], "and said nothing about the next") + assert.Equal(t, walker.interval, waits[2]) + + open, err := ledger.OpenLosses(ctx) + require.NoError(t, err) + require.Len(t, open, 1) + assert.Equal(t, loss.DeadlineAt.Add(15*time.Minute).UTC(), open[0].DeadlineAt.UTC(), + "the window moved out once, for the one wait the server asked for") +} + +// C2: a loss cannot live forever. However long a server keeps asking for +// patience, a loss closes at its absolute deadline with its ids reported. +func TestALossClosesAtItsAbsoluteDeadline(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, clock.at, 10*time.Minute, eventfeed.Filters{}) + require.NoError(t, err) + + walker, _ := newTestWalker(t, ledger, alwaysThrottling{}, clock) + walker.sleep = func(_ context.Context, d time.Duration) error { + clock.at = clock.at.Add(d) + return nil + } + require.NoError(t, walker.reconcile(ctx, loss)) + + open, err := ledger.OpenLosses(ctx) + require.NoError(t, err) + assert.Empty(t, open, "closed exactly once, at its absolute deadline") + unrecovered, err := ledger.UnrecoveredIDs(ctx) + require.NoError(t, err) + assert.Equal(t, []int64{17099838509}, unrecovered, "and its ids are reported, not forgotten") + assert.False(t, clock.at.Before(loss.DetectedAt.Add(maxLossLifetime)), "not before the absolute deadline") +} + +// A loss that was reconciled before its absolute deadline closes quietly: the +// error line is for ids nobody could serve, not for a tidy close. +func TestAReconciledLossPastItsDeadlineDoesNotReportUnrecovered(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, clock.at.Add(-48*time.Hour), time.Minute, eventfeed.Filters{}) + require.NoError(t, err) + // Served in the meantime, by the ordinary poll lane. + _, err = ledger.RecordSeen(ctx, testEvent(17099838509), LanePoll) + require.NoError(t, err) + + var logs bytes.Buffer + walker, _ := newTestWalker(t, ledger, &scriptedPolls{}, clock) + walker.log = slog.New(slog.NewTextHandler(&logs, nil)) + require.NoError(t, walker.reconcile(ctx, loss)) + + assert.NotContains(t, logs.String(), "unrecovered") + unrecovered, err := ledger.UnrecoveredIDs(ctx) + require.NoError(t, err) + assert.Empty(t, unrecovered) +} diff --git a/internal/connector/membership_test.go b/internal/connector/membership_test.go new file mode 100644 index 000000000..b5294eb29 --- /dev/null +++ b/internal/connector/membership_test.go @@ -0,0 +1,193 @@ +package connector + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" +) + +// failingFirstMembership fails its first read and serves buckets afterwards. +type failingFirstMembership struct { + mu sync.Mutex + calls atomic.Int32 + buckets []int64 +} + +func (m *failingFirstMembership) Buckets(context.Context) ([]int64, error) { + if m.calls.Add(1) == 1 { + return nil, errors.New("projects listing unavailable") + } + m.mu.Lock() + defer m.mu.Unlock() + return m.buckets, nil +} + +// E3: with the read at subscribe failed, the live subscription's buckets are +// unknown, not "everything". An event from a project the poll lane serves must +// still reconnect the live lane. +func TestAFailedFirstMembershipReadStillReconnectsForANewProject(t *testing.T) { + ledger := newTestLedger(t) + intake, transport, minter, polls := newFeedIntake(t, ledger, Options{ + Membership: &failingFirstMembership{buckets: []int64{48699913}}, + MembershipInterval: time.Hour, + }) + intake.membershipRetry = time.Hour + minter.ScriptTicket(ticket()) + minter.ScriptTicket(ticket()) + granted := testEvent(17099838600) + granted.BucketID = 777 + polls.ScriptPage(eventfeed.PollPage{Events: []eventfeed.Event{granted}, Position: "p1"}) + polls.ScriptPage(eventfeed.PollPage{Position: "p1"}) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := runInBackground(ctx, t, intake) + subscribedConn(t, transport) + + answered := 1 + require.Eventually(t, func() bool { + if conns := transport.Conns(); len(conns) > answered { + answerSubscription(conns[len(conns)-1]) + answered = len(conns) + } + return len(transport.Dials()) >= 2 + }, 5*time.Second, 10*time.Millisecond, "a project the subscription may not hold reconnects the live lane") + + // Learned once, it costs one reconnect, not one per event. + time.Sleep(100 * time.Millisecond) + assert.LessOrEqual(t, len(transport.Dials()), 2) + + cancel() + awaitReturn(t, done, "Run should return on shutdown") +} + +// E3: a failed read is retried on a backoff, not left for a full membership +// interval with no baseline to compare against. +func TestAFailedFirstMembershipReadIsRetriedSoon(t *testing.T) { + ledger := newTestLedger(t) + membership := &failingFirstMembership{buckets: []int64{48699913}} + intake, transport, minter, polls := newFeedIntake(t, ledger, Options{ + Membership: membership, + MembershipInterval: time.Hour, + }) + intake.membershipRetry = 20 * time.Millisecond + minter.ScriptTicket(ticket()) + polls.ScriptPage(eventfeed.PollPage{Position: "p1"}) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := runInBackground(ctx, t, intake) + subscribedConn(t, transport) + + require.Eventually(t, func() bool { return membership.calls.Load() >= 2 }, 2*time.Second, 10*time.Millisecond, + "the read is retried within its backoff, not after an hour") + require.Eventually(t, func() bool { + intake.mu.Lock() + defer intake.mu.Unlock() + return intake.listed != nil && intake.listed[48699913] + }, 2*time.Second, 10*time.Millisecond, "the first successful read becomes the baseline") + + cancel() + awaitReturn(t, done, "Run should return on shutdown") +} + +// E2/E3: while the project lister is failing, an unlisted project still costs +// one reconnect, not one per event. The buckets learned from arriving events +// are what the connector knows when it has no listing at all. +func TestAFailingListerStillCostsOneReconnectPerProject(t *testing.T) { + ledger := newTestLedger(t) + always := &alwaysFailingMembership{} + intake, transport, minter, polls := newFeedIntake(t, ledger, Options{ + Membership: always, + MembershipInterval: time.Hour, + }) + intake.membershipRetry = time.Hour + for range 10 { + minter.ScriptTicket(ticket()) + } + var events []eventfeed.Event + for id := int64(600); id < 605; id++ { + e := testEvent(id) + e.BucketID = 777 + events = append(events, e) + } + for range 10 { + polls.ScriptPage(eventfeed.PollPage{Events: events, Position: "p"}) + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := runInBackground(ctx, t, intake) + subscribedConn(t, transport) + + answered := 1 + deadline := time.Now().Add(time.Second) + for time.Now().Before(deadline) { + if conns := transport.Conns(); len(conns) > answered { + answerSubscription(conns[len(conns)-1]) + answered = len(conns) + } + time.Sleep(10 * time.Millisecond) + } + assert.LessOrEqual(t, len(transport.Dials()), 2, "five events from one project are one reconnect, lister or no lister") + + cancel() + awaitReturn(t, done, "Run should return on shutdown") +} + +type alwaysFailingMembership struct{} + +func (alwaysFailingMembership) Buckets(context.Context) ([]int64, error) { + return nil, errors.New("projects listing unavailable") +} + +// A bucket learned from an arriving event is provisional. The lister is the +// trust boundary: once it lists projects and does not name that one, the +// connector stops holding it, and the next event from it is unknown again. +func TestALearnedProjectIsRevokedWhenTheListerStopsNamingIt(t *testing.T) { + intake, _, _ := newTestIntake(t, nil, nil) + intake.opts.Membership = &flakyMembership{buckets: []int64{1}} + + require.False(t, intake.adoptListing([]int64{1}), "the first listing is a baseline") + intake.noteBucket(777) + intake.mu.Lock() + learned := intake.learned[777] + intake.mu.Unlock() + require.True(t, learned, "an event proved the project visible") + + // A later listing that does not name it takes it back. + intake.adoptListing([]int64{1}) + intake.mu.Lock() + stillHeld := intake.learned[777] || intake.listed[777] + intake.mu.Unlock() + assert.False(t, stillHeld, "the lister no longer names it, so the connector no longer holds it") + + // And its next event is unknown again, not silently trusted. Relearning + // is rate-limited per bucket, so the clock has to move past the + // membership interval for the next event to count as new. + intake.reconnectRequested() + require.Empty(t, intake.reconnect) + intake.opts.MembershipInterval = 0 + intake.noteBucket(777) + assert.Len(t, intake.reconnect, 1, "a revoked project's next event is unknown") +} + +// The listing the lister does name is what changes trigger a reconnect. +func TestOnlyTheListedSetDecidesAMembershipChange(t *testing.T) { + intake, _, _ := newTestIntake(t, nil, nil) + intake.opts.Membership = &flakyMembership{buckets: []int64{1, 2}} + + assert.False(t, intake.adoptListing([]int64{1, 2})) + intake.noteBucket(9) // learned, never listed + assert.False(t, intake.adoptListing([]int64{1, 2}), "a learned bucket the listing omits is not a change") + assert.True(t, intake.adoptListing([]int64{1}), "a listed bucket dropping off is a change") + assert.True(t, intake.adoptListing([]int64{1, 2}), "and its return is a change") +} diff --git a/internal/connector/ndjson/ndjson.go b/internal/connector/ndjson/ndjson.go new file mode 100644 index 000000000..d09d0dfe8 --- /dev/null +++ b/internal/connector/ndjson/ndjson.go @@ -0,0 +1,97 @@ +// Package ndjson writes the connector's stdout protocol: one JSON value per +// line, never torn. +// +// Intake's pointer lines and admission's verdict lines are both read by a +// process parsing the stream and watched by a person, so both need the same +// two guarantees. A line is written whole or reported failed, and concurrent +// writers never interleave inside a line. Callers sanitize API-controlled +// strings before encoding (richtext.SanitizeTerminal): JSON escapes C0 controls +// but passes C1 controls such as U+009B through as raw UTF-8. +package ndjson + +import ( + "encoding/json" + "fmt" + "io" + "reflect" + "sync" +) + +// Writer serializes whole lines onto w. The zero value with a nil w discards. +type Writer struct { + mu sync.Mutex + w io.Writer +} + +// NewWriter returns the Writer for w. A nil w discards every line. +// +// There is one Writer, and so one lock, per sink for the life of the process. +// Intake and admission each build their writer from the stdout they are +// handed; if those were two Writers their locks would not coordinate, and a +// pointer line and a verdict line could tear each other inside one write. +// Keying on the sink makes that impossible whoever calls this, without every +// caller having to agree to pass one instance around. +// +// A value-typed sink is the one case this cannot settle by itself: two copies +// can wrap one underlying writer and compare unequal, and hashing one can +// panic, so each gets its own Writer. A composition that puts two producers on +// one such sink passes them one Writer instead — intake's Options.Lines and +// admission's RunOptions.LineWriter are there for exactly that. +// +// Only reference-like sinks are keyed — a pointer, a channel, an unsafe +// pointer — which covers every real one, os.Stdout included. A value-typed +// sink is a copy rather than the same sink, and hashing one can panic when it +// holds an uncomparable field, so it gets a Writer of its own. The map holds +// one entry per distinct sink for the life of the process; a connector has +// one stdout. +func NewWriter(w io.Writer) *Writer { + if w == nil { + return &Writer{} + } + switch reflect.TypeOf(w).Kind() { + case reflect.Pointer, reflect.Chan, reflect.UnsafePointer: + default: + return &Writer{w: w} + } + sinksMu.Lock() + defer sinksMu.Unlock() + if existing, ok := sinks[w]; ok { + return existing + } + writer := &Writer{w: w} + sinks[w] = writer + return writer +} + +var ( + sinksMu sync.Mutex + sinks = map[io.Writer]*Writer{} +) + +// WriteLine encodes v and writes it followed by a newline, whole. +// +// A writer that takes part of the line is written the rest, and one that takes +// none without an error has failed (io.ErrShortWrite): a torn line is worse +// than no line to whoever parses the stream. +func (l *Writer) WriteLine(v any) error { + if l == nil || l.w == nil { + return nil + } + b, err := json.Marshal(v) + if err != nil { + return fmt.Errorf("encode line: %w", err) + } + l.mu.Lock() + defer l.mu.Unlock() + for rest := append(b, '\n'); len(rest) > 0; { + n, err := l.w.Write(rest) + if err != nil { + return fmt.Errorf("write line: %w", err) + } + if n <= 0 { + return fmt.Errorf("write line: %w", io.ErrShortWrite) + } + rest = rest[n:] + } + return nil +} diff --git a/internal/connector/ndjson/ndjson_test.go b/internal/connector/ndjson/ndjson_test.go new file mode 100644 index 000000000..3b5421d18 --- /dev/null +++ b/internal/connector/ndjson/ndjson_test.go @@ -0,0 +1,37 @@ +package ndjson + +import ( + "bytes" + "io" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type threeBytes struct{ bytes.Buffer } + +func (w *threeBytes) Write(p []byte) (int, error) { + if len(p) > 3 { + p = p[:3] + } + return w.Buffer.Write(p) +} + +type stuck struct{} + +func (stuck) Write([]byte) (int, error) { return 0, nil } + +func TestWriteLineCompletesShortWrites(t *testing.T) { + var w threeBytes + require.NoError(t, NewWriter(&w).WriteLine(map[string]int{"event_id": 42})) + assert.Equal(t, "{\"event_id\":42}\n", w.String()) +} + +func TestWriteLineReportsAWriterThatTakesNothing(t *testing.T) { + assert.ErrorIs(t, NewWriter(stuck{}).WriteLine(1), io.ErrShortWrite) +} + +func TestWriteLineWithNoWriterDiscards(t *testing.T) { + assert.NoError(t, NewWriter(nil).WriteLine(1)) +} diff --git a/internal/connector/ndjson/shared_test.go b/internal/connector/ndjson/shared_test.go new file mode 100644 index 000000000..880cf4cfc --- /dev/null +++ b/internal/connector/ndjson/shared_test.go @@ -0,0 +1,81 @@ +package ndjson + +import ( + "bufio" + "bytes" + "encoding/json" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// lockedShortWriter is a sink that is safe for concurrent use but takes at +// most three bytes per call, so two writers that do not share a lock can +// interleave inside a line. +type lockedShortWriter struct { + mu sync.Mutex + buf bytes.Buffer +} + +func (w *lockedShortWriter) Write(p []byte) (int, error) { + w.mu.Lock() + defer w.mu.Unlock() + if len(p) > 3 { + p = p[:3] + } + return w.buf.Write(p) +} + +// One process stdout has one lock. Intake and admission each build their writer +// for the same sink; those writers must be one writer, or their lines tear +// each other. +func TestWritersForOneSinkShareOneLock(t *testing.T) { + sink := &lockedShortWriter{} + intake, admission := NewWriter(sink), NewWriter(sink) + + var wg sync.WaitGroup + for _, w := range []*Writer{intake, admission} { + wg.Add(1) + go func() { + defer wg.Done() + for i := range 300 { + require.NoError(t, w.WriteLine(map[string]any{"line": i, "padding": "0123456789abcdef"})) + } + }() + } + wg.Wait() + + scanner := bufio.NewScanner(bytes.NewReader(sink.buf.Bytes())) + lines := 0 + for scanner.Scan() { + var v map[string]any + require.NoError(t, json.Unmarshal(scanner.Bytes(), &v), "line %d is not whole JSON: %q", lines, scanner.Text()) + lines++ + } + assert.Equal(t, 600, lines) +} + +func TestWritersForDifferentSinksAreIndependent(t *testing.T) { + var a, b bytes.Buffer + assert.NotSame(t, NewWriter(&a), NewWriter(&b)) + assert.Same(t, NewWriter(&a), NewWriter(&a)) +} + +// A sink whose type is comparable but whose fields are not panics when hashed. +// Only reference-like sinks are keyed, so such a sink never reaches the map. +type wrapperSink struct { + inner any + buf *bytes.Buffer +} + +func (w wrapperSink) Write(p []byte) (int, error) { return w.buf.Write(p) } + +func TestAnUnhashableSinkIsNotKeyed(t *testing.T) { + sink := wrapperSink{inner: []int{1, 2, 3}, buf: &bytes.Buffer{}} + require.NotPanics(t, func() { + require.NoError(t, NewWriter(sink).WriteLine(map[string]int{"line": 1})) + }) + assert.Equal(t, "{\"line\":1}\n", sink.buf.String()) +} diff --git a/internal/connector/pointer_test.go b/internal/connector/pointer_test.go new file mode 100644 index 000000000..c92ac0ecc --- /dev/null +++ b/internal/connector/pointer_test.go @@ -0,0 +1,67 @@ +package connector + +import ( + "bytes" + "context" + "encoding/json" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Control characters built from their code points, so the canaries are +// unmistakable in review. +var ( + esc = string(rune(0x1b)) + bel = string(rune(0x07)) + csi = string(rune(0x9b)) // C1 Control Sequence Introducer + osc = string(rune(0x9d)) // C1 Operating System Command + st = string(rune(0x9c)) // C1 String Terminator +) + +// The pointer line is a wire, and it is also what a person watching the +// connector sees. The event type, kind and action come from Basecamp; JSON +// escapes C0 controls but passes C1 controls such as CSI through as raw UTF-8, +// which a terminal executes. +func TestPointerLineCarriesNoTerminalControls(t *testing.T) { + var out bytes.Buffer + intake, _, _ := newTestIntake(t, nil, &out) + event := testEvent(1) + event.EventType = "comment.created" + csi + "31m" + esc + "]0;owned" + bel + event.Kind = "comment_created" + esc + "[2J" + event.Action = "created" + osc + "8;;evil" + st + require.NoError(t, intake.ingest(context.Background(), event, LanePoll)) + + line := out.String() + for name, control := range map[string]string{"ESC": esc, "CSI": csi, "OSC": osc, "ST": st, "BEL": bel} { + assert.NotContains(t, line, control, name) + } + var pointer Pointer + require.NoError(t, json.Unmarshal([]byte(strings.TrimSpace(line)), &pointer)) + assert.True(t, strings.HasPrefix(pointer.EventType, "comment.created")) +} + +// shortWriter accepts at most three bytes per call. +type shortWriter struct{ bytes.Buffer } + +func (w *shortWriter) Write(p []byte) (int, error) { + if len(p) > 3 { + p = p[:3] + } + return w.Buffer.Write(p) +} + +// A writer that takes part of a line is written the rest: a torn line is worse +// than no line to whoever parses the stream. +func TestPointerLineIsWrittenWholeThroughShortWrites(t *testing.T) { + var out shortWriter + intake, _, _ := newTestIntake(t, nil, &out) + require.NoError(t, intake.ingest(context.Background(), testEvent(1), LanePoll)) + + var pointer Pointer + require.NoError(t, json.Unmarshal(bytes.TrimSpace(out.Bytes()), &pointer)) + assert.Equal(t, int64(1), pointer.EventID) + assert.True(t, bytes.HasSuffix(out.Bytes(), []byte("\n"))) +} diff --git a/internal/connector/queue.go b/internal/connector/queue.go new file mode 100644 index 000000000..37132d094 --- /dev/null +++ b/internal/connector/queue.go @@ -0,0 +1,327 @@ +package connector + +import ( + "context" + "errors" + "fmt" + "log/slog" + "sync" + + "github.com/basecamp/basecamp-cli/internal/richtext" +) + +// Backlog thresholds. Intake is the only work on the feed's delivery path, so +// the queue is where a slow admission shows up. +const ( + // DefaultBacklogWarn is the depth at which the backlog is worth saying out + // loud. Nothing changes; the operator is told. + DefaultBacklogWarn = 1_000 + // DefaultBacklogPause is the depth at which intake stops consuming the + // feed. The checkpoint does not move while it is paused, because the + // package saves a position only after its page's events were accepted and + // intake has stopped accepting them — so a crash while paused resumes from + // before the backlog rather than after it. + DefaultBacklogPause = 10_000 +) + +// Queue is the seam between intake and admission: intake writes a pointer and +// hands over an id, admission reads it when it gets there. Two queues with +// visible depth rather than one pipeline, so a busy dispatcher is absorbed up +// to the pause threshold instead of being felt on the socket — and so the +// place where work is piling up is the place the depth is showing. At that +// threshold the offer waits: the backlog is bounded, and the pause is the +// backpressure reaching the feed, reported rather than hidden. +type Queue struct { + ids chan int64 + warnAt int + + // edges serializes the depth sample with the warning transition and its + // callback. Unserialized, an offer can sample a warning depth, a take can + // drain and find nothing to recover from, and the offer then raises a + // warning that no later event will clear. + edges sync.Mutex + warned bool + // depth is the queue's length as the operations themselves report it. + depth int + // afterChannelOp runs between a channel operation and the transition it + // produces. A test seam: it is where the queue was losing a crossing. + afterChannelOp func() + // pending holds edges decided but not yet delivered, in the order they + // were decided; delivering says a goroutine is already draining them. + pending []queueEdge + delivering bool + // waiting counts offers blocked for room. Intake and every open repair + // walk offer concurrently, so a single flag would be cleared by the first + // waiter to resume while another — perhaps the feed — still waits. It is + // held under edges with the depth, because its callbacks go through the + // same ordered drain: told out of order, a resume that started first but + // finished last leaves the operator reading "resumed" while the feed is + // paused. + waiting int + paused bool + + // OnWarn fires when the depth first crosses the warning threshold, and + // OnRecover when it falls back below. Both are optional. + OnWarn func(depth int) + OnRecover func(depth int) + // OnPause fires when an offer begins waiting for room, and OnResume when + // it stops. The feed is not being consumed in between. + OnPause func(depth int) + OnResume func(depth int) + // log reports a callback that panicked. Held under edges: a queue can be + // adopted by intake while another component is already using it, and a + // field written by one and read by the other is a race. + log *slog.Logger +} + +// SetLogger sets where a callback that panicked is reported. Safe to call +// while the queue is in use. +func (q *Queue) SetLogger(log *slog.Logger) { + q.edges.Lock() + defer q.edges.Unlock() + q.log = log +} + +// adoptLogger sets log only if nobody has set one, as one step: a check and a +// set taken apart would let two adopters both find it unset. +func (q *Queue) adoptLogger(log *slog.Logger) { + q.edges.Lock() + defer q.edges.Unlock() + if q.log == nil { + q.log = log + } +} + +// NewQueue builds a queue that warns at warnAt and pauses the feed at pauseAt. +func NewQueue(warnAt, pauseAt int) (*Queue, error) { + if pauseAt <= 0 { + return nil, errors.New("connector: queue pause threshold must be positive") + } + if warnAt <= 0 || warnAt > pauseAt { + return nil, errors.New("connector: queue warning threshold must be positive and no higher than the pause threshold") + } + // Capacity IS the pause threshold: a full channel is a blocked offer is a + // feed that has stopped being read. There is no second mechanism to keep + // in agreement with this one. + return &Queue{ids: make(chan int64, pauseAt), warnAt: warnAt}, nil +} + +// Offer hands an event id to admission, waiting for room when the backlog is +// at the pause threshold. Waiting here is the pause: the caller is the feed's +// delivery path, and it is not reading the feed while it waits. +func (q *Queue) Offer(ctx context.Context, id int64) error { + // Counted before it is sent. An id on its way into the queue is backlog + // either way, and counting it first is what keeps the depth honest: a + // take can only receive what a send has already put in, so its decrement + // can never be applied before the increment it belongs to, and a crossing + // can never be lost to the order two operations happen to take the lock in. + q.stage(1) + // An offer that leaves without sending must take its count back with it, + // on every way out — the ordinary cancellation, and an unwinding this + // function did not choose. A count left behind is a queue that reports an + // item nobody can take and, at the threshold, a pause nothing can lift. + settled := false + settle := func(sent bool) { + if settled { + return + } + settled = true + if !sent { + q.stage(-1) + q.deliver() + } + } + defer func() { settle(false) }() + + select { + case q.ids <- id: + settle(true) + q.afterOp() + q.deliver() + return nil + default: + } + + q.stageWait(1) + // Registered BEFORE the pause is delivered: the wait is recorded, and + // what ends it must already be in place when anything else runs. + defer func() { + q.stageWait(-1) + q.deliver() + }() + q.deliver() + + select { + case q.ids <- id: + settle(true) + q.afterOp() + q.deliver() + return nil + case <-ctx.Done(): + // It never went in, so it is not backlog. Taken back here rather than + // left to the defer, so the resume that follows reports the depth + // without it. + settle(false) + return ctx.Err() + } +} + +// Take returns the next id, waiting for one until ctx ends. +// +// There is deliberately no Close. Shutdown is the context: a closed channel +// would turn every offer racing the close into a panic, and nothing here needs +// a drained-and-done signal that cancellation does not already give. +func (q *Queue) Take(ctx context.Context) (int64, error) { + select { + case id := <-q.ids: + q.afterOp() + q.stage(-1) + q.deliver() + return id, nil + case <-ctx.Done(): + return 0, ctx.Err() + } +} + +// Depth is the number of ids waiting. +func (q *Queue) Depth() int { + q.edges.Lock() + defer q.edges.Unlock() + return q.depth +} + +// Paused reports whether an offer is currently waiting for room — which is to +// say whether the feed is being consumed. +func (q *Queue) Paused() bool { + q.edges.Lock() + defer q.edges.Unlock() + return q.waiting > 0 +} + +// afterOp is a test seam: it runs between a channel operation and whatever +// follows it, which is where a crossing used to be lost. +func (q *Queue) afterOp() { + if q.afterChannelOp != nil { + q.afterChannelOp() + } +} + +// stage records what one operation does to the depth and decides the edges it +// crosses, without delivering them. +// +// The depth is a counter this method owns, not a later read of the channel's +// length. With a length read, an offer and a concurrent take could both +// observe the depth AFTER the take, and a queue that really crossed the +// threshold raised no warning at all. Every delta is applied under one lock +// and the crossing is derived from the depth that operation produced. +// +// Deciding and delivering are separate because an offer counts its id before +// it sends it: a callback delivered there could call Take and wait for an id +// the offer has not sent yet, and neither would ever finish. So the edges wait +// for deliver, which every operation calls once its channel work is done. +func (q *Queue) stage(delta int) { + q.edges.Lock() + defer q.edges.Unlock() + q.depth += delta + depth := q.depth + switch { + case depth >= q.warnAt && !q.warned: + q.warned = true + q.pending = append(q.pending, queueEdge{fire: q.OnWarn, depth: depth}) + case depth < q.warnAt && q.warned: + q.warned = false + q.pending = append(q.pending, queueEdge{fire: q.OnRecover, depth: depth}) + } +} + +// stageWait records an offer starting or finishing its wait for room and the +// pause edge that crosses, without delivering it. +// +// Pause and resume are transitions of the same state as the warning edges, so +// they are decided under the same lock and delivered by the same drain, in the +// order they were decided. Fired from the waiting goroutines themselves they +// could interleave: a resume delayed in its callback could land after a later +// pause, and an observer would be left believing the feed is being read when +// it is not. +func (q *Queue) stageWait(delta int) { + q.edges.Lock() + defer q.edges.Unlock() + q.waiting += delta + switch { + case q.waiting > 0 && !q.paused: + q.paused = true + q.pending = append(q.pending, queueEdge{fire: q.OnPause, depth: q.depth}) + case q.waiting == 0 && q.paused: + q.paused = false + q.pending = append(q.pending, queueEdge{fire: q.OnResume, depth: q.depth}) + } +} + +// deliver hands the staged edges to their callbacks, in the order they were +// decided, one goroutine at a time and with the lock released around each: a +// callback may use the queue, and its own edge is delivered by the drain +// already running. +func (q *Queue) deliver() { + q.edges.Lock() + if q.delivering { + q.edges.Unlock() + return + } + q.delivering = true + // Cleared with a defer: a callback that panics and is recovered above + // would otherwise leave the drain latched and every later edge + // undelivered. The loop below holds the lock whenever it is not inside a + // callback, so this runs holding it on every path. + defer func() { + q.delivering = false + q.edges.Unlock() + }() + for len(q.pending) > 0 { + edge := q.pending[0] + q.pending = q.pending[1:] + q.edges.Unlock() + // The lock is retaken by a defer, not after the call: fire contains a + // panic, but a future change to it must not be able to leave this + // loop, or the cleanup above, holding nothing. + func() { + defer q.edges.Lock() + q.fire(edge) + }() + } +} + +// fire runs one callback with the lock released and a panic contained. +// +// The callbacks belong to whoever built the queue, and the goroutine they run +// on is the feed's delivery path or a worker taking work off it. A callback +// that panics is a reporting bug; it is not a reason to lose the connection, +// and it must not leave the queue part-way through an update. The state the +// callback is told about was committed before it ran, so what it does cannot +// change it. +func (q *Queue) fire(edge queueEdge) { + defer func() { + if p := recover(); p != nil { + q.logger().Error("a backlog callback panicked; the queue carried on without it", + "panic", richtext.SanitizeSingleLine(fmt.Sprint(p))) + } + }() + if edge.fire != nil { + edge.fire(edge.depth) + } +} + +// logger is read under edges, so it must never be called holding them. Its one +// caller, fire, runs with the lock released. +func (q *Queue) logger() *slog.Logger { + q.edges.Lock() + defer q.edges.Unlock() + if q.log != nil { + return q.log + } + return slog.New(slog.DiscardHandler) +} + +type queueEdge struct { + fire func(int) + depth int +} diff --git a/internal/connector/queue_test.go b/internal/connector/queue_test.go new file mode 100644 index 000000000..8a764b063 --- /dev/null +++ b/internal/connector/queue_test.go @@ -0,0 +1,301 @@ +package connector + +import ( + "bytes" + "context" + "log/slog" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestQueueWarnsOnceOnTheWayUpAndRecoversOnTheWayDown(t *testing.T) { + queue, err := NewQueue(2, 4) + require.NoError(t, err) + + var warnings, recoveries int + queue.OnWarn = func(int) { warnings++ } + queue.OnRecover = func(int) { recoveries++ } + + ctx := context.Background() + require.NoError(t, queue.Offer(ctx, 1)) + assert.Zero(t, warnings) + + require.NoError(t, queue.Offer(ctx, 2)) + require.NoError(t, queue.Offer(ctx, 3)) + assert.Equal(t, 1, warnings, "a backlog that sits above the threshold is one warning, not one per event") + + _, err = queue.Take(ctx) + require.NoError(t, err) + _, err = queue.Take(ctx) + require.NoError(t, err) + assert.Equal(t, 1, recoveries, "a warning must not be left standing after the backlog drained") +} + +// At the pause threshold intake stops consuming the feed. Because the feed +// saves a position only after its page's events were accepted, and intake has +// stopped accepting them, the checkpoint stops moving too — which is what makes +// the pause safe rather than a way to lose the backlog. +func TestQueuePausesTheCallerAtTheThreshold(t *testing.T) { + queue, err := NewQueue(1, 2) + require.NoError(t, err) + + paused := make(chan int, 1) + queue.OnPause = func(depth int) { paused <- depth } + + ctx := context.Background() + require.NoError(t, queue.Offer(ctx, 1)) + require.NoError(t, queue.Offer(ctx, 2)) + assert.Equal(t, 2, queue.Depth()) + + done := make(chan error, 1) + go func() { done <- queue.Offer(ctx, 3) }() + + select { + case depth := <-paused: + // Three: the two in the queue and the one waiting to go in, which is + // backlog too. + assert.Equal(t, 3, depth) + case <-time.After(2 * time.Second): + t.Fatal("the third offer should have waited for room") + } + assert.True(t, queue.Paused()) + + select { + case err := <-done: + t.Fatalf("the third offer returned while the queue was full: %v", err) + case <-time.After(50 * time.Millisecond): + } + + _, err = queue.Take(ctx) + require.NoError(t, err) + + select { + case err := <-done: + require.NoError(t, err) + case <-time.After(2 * time.Second): + t.Fatal("the offer should have resumed once there was room") + } +} + +func TestQueueOfferHonoursCancellation(t *testing.T) { + queue, err := NewQueue(1, 1) + require.NoError(t, err) + require.NoError(t, queue.Offer(context.Background(), 1)) + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- queue.Offer(ctx, 2) }() + time.Sleep(20 * time.Millisecond) + cancel() + + select { + case err := <-done: + assert.ErrorIs(t, err, context.Canceled) + case <-time.After(2 * time.Second): + t.Fatal("a paused offer must unblock on shutdown") + } +} + +func TestQueueRefusesNonsenseThresholds(t *testing.T) { + _, err := NewQueue(10, 5) + assert.Error(t, err) + _, err = NewQueue(0, 5) + assert.Error(t, err) + _, err = NewQueue(5, 0) + assert.Error(t, err) +} + +// F2: a crossing is never lost. An offer and a concurrent take could both read +// the depth after the take, so a queue that really crossed the threshold +// raised no warning at all. +func TestACrossingIsNotLostToAConcurrentTake(t *testing.T) { + queue, err := NewQueue(1, 4) + require.NoError(t, err) + var warns, recovers int + var mu sync.Mutex + queue.OnWarn = func(int) { mu.Lock(); warns++; mu.Unlock() } + queue.OnRecover = func(int) { mu.Lock(); recovers++; mu.Unlock() } + + ctx := context.Background() + taken := make(chan int64, 1) + // The take lands between the offer's send and the transition it produces. + // A flag, not a Once: the take runs this hook too, and must not wait on + // the offer that is waiting for it. + var interleaved atomic.Bool + queue.afterChannelOp = func() { + if !interleaved.CompareAndSwap(false, true) { + return + } + done := make(chan struct{}) + go func() { + defer close(done) + id, err := queue.Take(ctx) + assert.NoError(t, err) + taken <- id + }() + <-done + } + + require.NoError(t, queue.Offer(ctx, 42)) + assert.Equal(t, int64(42), <-taken) + + mu.Lock() + defer mu.Unlock() + assert.Equal(t, 1, warns, "the queue crossed the threshold, so it warned") + assert.Equal(t, 1, recovers, "and the take brought it back") + assert.Zero(t, queue.Depth()) +} + +// A callback belongs to whoever built the queue, and a panic in one is a +// reporting bug — not a reason to lose the feed. It is contained, and the +// queue's own state is correct afterwards because it was committed before the +// callback ran. +func TestAPanickingBacklogCallbackLeavesTheQueueUsable(t *testing.T) { + queue, err := NewQueue(1, 4) + require.NoError(t, err) + + var warnings, recoveries int + queue.OnWarn = func(int) { warnings++; panic("a warning callback panics") } + queue.OnRecover = func(int) { recoveries++ } + + ctx := context.Background() + require.NoError(t, queue.Offer(ctx, 1), "the callback's panic is not the offer's failure") + assert.Equal(t, 1, warnings) + + // The drain is unlatched and the queue still works: a second id goes in, + // both come out, and the recovery edge is delivered. + require.NoError(t, queue.Offer(ctx, 2)) + assert.Equal(t, 2, queue.Depth()) + + first, err := queue.Take(ctx) + require.NoError(t, err) + assert.Equal(t, int64(1), first) + second, err := queue.Take(ctx) + require.NoError(t, err) + assert.Equal(t, int64(2), second) + assert.Equal(t, 0, queue.Depth()) + assert.Equal(t, 1, recoveries, "the drain should still be delivering edges") +} + +// An operator watching the queue must never be told the feed resumed while it +// is paused. The transitions are delivered in the order they happened, even +// when an earlier callback is slower than a later transition. +func TestPauseAndResumeAreObservedInTheOrderTheyHappened(t *testing.T) { + queue, err := NewQueue(1, 1) + require.NoError(t, err) + + var mu sync.Mutex + var seen []string + record := func(what string) { + mu.Lock() + defer mu.Unlock() + seen = append(seen, what) + } + observed := func() []string { + mu.Lock() + defer mu.Unlock() + return append([]string(nil), seen...) + } + resumeStarted, holdResume := make(chan struct{}), make(chan struct{}) + queue.OnPause = func(int) { record("paused") } + var firstResume sync.Once + queue.OnResume = func(int) { + held := false + firstResume.Do(func() { held = true; close(resumeStarted) }) + if held { + <-holdResume + } + record("resumed") + } + + ctx := context.Background() + require.NoError(t, queue.Offer(ctx, 1)) + + var waiters sync.WaitGroup + waiters.Add(1) + go func() { + defer waiters.Done() + assert.NoError(t, queue.Offer(ctx, 2)) // waits for room, then resumes + }() + require.Eventually(t, queue.Paused, time.Second, time.Millisecond) + + first, err := queue.Take(ctx) + require.NoError(t, err) + assert.Equal(t, int64(1), first) + <-resumeStarted // the resume has begun and is not finished + + waiters.Add(1) + go func() { + defer waiters.Done() + assert.NoError(t, queue.Offer(ctx, 3)) // pauses again, mid-resume + }() + require.Eventually(t, queue.Paused, time.Second, time.Millisecond) + + close(holdResume) + require.Eventually(t, func() bool { return len(observed()) == 3 }, time.Second, time.Millisecond) + assert.Equal(t, []string{"paused", "resumed", "paused"}, observed(), + "a resume that started first must not be reported after the pause that followed it") + + second, err := queue.Take(ctx) + require.NoError(t, err) + assert.Equal(t, int64(2), second) + waiters.Wait() + third, err := queue.Take(ctx) + require.NoError(t, err) + assert.Equal(t, int64(3), third) + assert.False(t, queue.Paused()) +} + +// A pause callback that panics must leave nothing behind: the id it was +// waiting for still goes in, the depth is the queue's real depth, the pause +// lifts, and the next crossing is reported. +func TestAPanickingPauseCallbackLeavesNoPhantomBacklog(t *testing.T) { + queue, err := NewQueue(1, 1) + require.NoError(t, err) + + var logs bytes.Buffer + queue.SetLogger(slog.New(slog.NewTextHandler(&logs, nil))) + var pauses, resumes atomic.Int32 + queue.OnPause = func(int) { + if pauses.Add(1) == 1 { + panic("a pause callback panics") + } + } + queue.OnResume = func(int) { resumes.Add(1) } + + ctx := context.Background() + require.NoError(t, queue.Offer(ctx, 1)) + + waiting := make(chan error, 1) + go func() { waiting <- queue.Offer(ctx, 2) }() + require.Eventually(t, queue.Paused, time.Second, time.Millisecond) + + first, err := queue.Take(ctx) + require.NoError(t, err) + assert.Equal(t, int64(1), first) + require.NoError(t, <-waiting, "the offer the panicking callback announced still completes") + + assert.Equal(t, 1, queue.Depth(), "no phantom item") + assert.False(t, queue.Paused(), "no phantom pause") + assert.Contains(t, logs.String(), "a backlog callback panicked") + + // And the next crossing is still reported, on both sides. + second, err := queue.Take(ctx) + require.NoError(t, err) + assert.Equal(t, int64(2), second) + require.NoError(t, queue.Offer(ctx, 3)) + blocked := make(chan error, 1) + go func() { blocked <- queue.Offer(ctx, 4) }() + require.Eventually(t, queue.Paused, time.Second, time.Millisecond) + _, err = queue.Take(ctx) + require.NoError(t, err) + require.NoError(t, <-blocked) + assert.Equal(t, int32(2), pauses.Load()) + assert.Equal(t, int32(2), resumes.Load()) + assert.Equal(t, 1, queue.Depth()) +} diff --git a/internal/connector/repair.go b/internal/connector/repair.go new file mode 100644 index 000000000..caa006ab7 --- /dev/null +++ b/internal/connector/repair.go @@ -0,0 +1,535 @@ +package connector + +import ( + "context" + "errors" + "log/slog" + "net/url" + "strconv" + "time" + + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" +) + +// repairWalker reconciles one accepted buffer overflow. +// +// It repairs with EXPLICIT IDS and never with the feed's position. The +// position is an opaque signed token, so nothing can be compared to it; and +// the overflow's ids are live ids, far ahead of the poll lane's safety delay, +// so a checkpoint taken from one would skip everything behind it that the +// delay had not yet served. The walk therefore runs on its own cursor, kept on +// the loss record, and the feed's checkpoint is never written from it. +// +// It runs off the delivery path. Nothing about live intake waits for it. +type repairWalker struct { + ledger *Ledger + polls eventfeed.PollSource + // maxPages caps the pages one pass walks; zero means maxRepairPagesPerPass. + maxPages int + // retryAfter is a server-directed wait the last pass was given. It is + // honored exactly and is exempt from the repair cadence's own cap. + retryAfter time.Duration + // origin is the API origin every URL the walk follows must stay on. + origin string + filters eventfeed.Filters + ingest func(ctx context.Context, event eventfeed.Event, lane Lane) error + now func() time.Time + interval time.Duration + log *slog.Logger + + // sleep is the wait between repair polls; nil means time.After. Injected + // so a test can run ten minutes of repair polls without taking ten + // minutes. + sleep func(ctx context.Context, d time.Duration) error +} + +// reconcileLoss walks a loss under its own recorded filter set. +func (w *repairWalker) reconcileLoss(ctx context.Context, loss Loss) error { + walker := *w + if loss.HasFilters { + // Recorded with the loss, empty set included: the whole-account feed + // is a filter set, and walking it narrowly would miss the very events + // the loss is about. + walker.filters = loss.Filters + } + return walker.reconcile(ctx, loss) +} + +func (w *repairWalker) reconcile(ctx context.Context, loss Loss) error { + for { + if !w.now().Before(loss.DetectedAt.Add(maxLossLifetime)) { + // The absolute deadline. A throttle postpones the window, and a + // server that keeps asking for patience could postpone it forever; + // a loss that can never close is its own kind of silence. At a day + // old it closes, whatever the retry state, with its ids reported. + unrecovered, err := w.ledger.CloseLoss(ctx, loss.ID, w.now()) + if err != nil { + return err + } + if unrecovered > 0 { + w.log.Error("a buffer overflow reached its absolute deadline; what is still missing is unrecovered", + "loss_id", loss.ID, "unrecovered", unrecovered, "age", w.now().Sub(loss.DetectedAt)) + } else { + w.log.Info("a buffer overflow was reconciled before its absolute deadline", + "loss_id", loss.ID, "age", w.now().Sub(loss.DetectedAt)) + } + return nil + } + if !w.now().Before(loss.DeadlineAt) { + // Past the window already — perhaps across restarts whose walks + // each ended in a failure no retry fixes. One last pass is still + // worth trying; after it the loss closes, unless the server + // refused that attempt too. + return w.finalPass(ctx, &loss) + } + + done, err := w.settled(ctx, loss) + if err != nil || done { + return err + } + + cursor, err := w.walk(ctx, &loss) + if errors.Is(err, errReconciliationEnded) { + return w.closeIfExpired(ctx, loss) + } + if err != nil { + return err + } + loss.RepairCursor = cursor + + // Re-read before waiting: a walk that served everything has nothing + // left to wait for. + done, err = w.settled(ctx, loss) + if err != nil || done { + return err + } + + // A missing id the walk did not serve is NOT yet a gap: `next` can end + // while an event is still inside the poll lane's safety delay. The + // walk repeats on the repair cadence until the window closes — or + // after a server-directed wait, whichever is longer. + wait := w.interval + if w.retryAfter > wait { + wait = w.retryAfter + } + if w.retryAfter > 0 { + if err := w.postpone(ctx, &loss); err != nil { + return err + } + // Used: this wait was the server's answer to this attempt, and + // says nothing about the next one. + w.retryAfter = 0 + } + if err := w.waitWithin(ctx, wait, loss); err != nil { + return err + } + } +} + +// waitWithin sleeps for wait, but never past the loss's absolute deadline. +// +// The 24-hour lifetime bounds the LOSS, not the intervals between its +// attempts, and a server-directed Retry-After is not bounded by anything: one +// response asking for thirty hours would otherwise hold the loss open — and +// its repair worker with it — well past the deadline the connector promises. +// Woken at the deadline, the caller's next turn around the loop sees it and +// closes the loss without polling first, which is right: the server asked for +// a wait this connector no longer has to give. +func (w *repairWalker) waitWithin(ctx context.Context, wait time.Duration, loss Loss) error { + if remaining := loss.DetectedAt.Add(maxLossLifetime).Sub(w.now()); remaining < wait { + wait = remaining + } + if wait <= 0 { + return nil + } + return w.wait(ctx, wait) +} + +// postpone moves a loss's window out by the wait the server asked for. +// +// This is the whole rule for a throttle, in one place: it is not a failure and +// not a verdict, and the time it costs belongs to the server, not to the loss. +// The window therefore never runs out while a server is asking for patience, +// and a throttled attempt never condemns an id. +func (w *repairWalker) postpone(ctx context.Context, loss *Loss) error { + extended := loss.DeadlineAt.Add(w.retryAfter) + if err := w.ledger.ExtendLossDeadline(ctx, loss.ID, extended); err != nil { + return err + } + loss.DeadlineAt = extended + return nil +} + +// finalPass runs one walk for a loss past its window and then closes it, +// whatever the walk managed: its missing ids become unrecovered. +func (w *repairWalker) finalPass(ctx context.Context, loss *Loss) error { + if done, err := w.settled(ctx, *loss); err != nil || done { + return err + } + if _, err := w.walk(ctx, loss); err != nil && !errors.Is(err, errReconciliationEnded) { + return err + } + if done, err := w.settled(ctx, *loss); err != nil || done { + return err + } + if w.retryAfter > 0 { + // The server refused this attempt and named a wait, so there is + // nothing to conclude about the ids. The window moves out by the wait + // and the loss is walked again. + wait := w.retryAfter + if err := w.postpone(ctx, loss); err != nil { + return err + } + w.retryAfter = 0 + w.log.Warn("the last repair attempt was throttled; waiting as the server asked and trying again", + "loss_id", loss.ID, "retry_after", wait) + if err := w.waitWithin(ctx, wait, *loss); err != nil { + return err + } + return w.reconcile(ctx, *loss) + } + if err := ctx.Err(); err != nil { + // The final attempt did not run to its end; closing now would condemn + // ids on a pass that never happened. + return err + } + if done, err := w.settled(ctx, *loss); err != nil || done { + return err + } + return w.closeExpired(ctx, *loss) +} + +// closeIfExpired closes a loss whose walk ended early, if its window is over. +// A walk that ends in a failure no retry fixes leaves the loss open for the +// next start — but not forever. +func (w *repairWalker) closeIfExpired(ctx context.Context, loss Loss) error { + if err := ctx.Err(); err != nil { + return err + } + if w.now().Before(loss.DeadlineAt) { + return nil + } + return w.closeExpired(ctx, loss) +} + +func (w *repairWalker) closeExpired(ctx context.Context, loss Loss) error { + unrecovered, err := w.ledger.CloseLoss(ctx, loss.ID, w.now()) + if err != nil { + return err + } + if unrecovered > 0 { + w.log.Error("a buffer overflow's window closed with events unrecovered", "loss_id", loss.ID, "unrecovered", unrecovered) + } + return nil +} + +// settled closes the loss and reports true when nothing is missing any more. +func (w *repairWalker) settled(ctx context.Context, loss Loss) (bool, error) { + missing, err := w.ledger.MissingIDs(ctx, loss.ID, LossMissing) + if err != nil { + return false, err + } + if len(missing) > 0 { + return false, nil + } + if _, err := w.ledger.CloseLoss(ctx, loss.ID, w.now()); err != nil { + return false, err + } + // "Nothing missing" is not "everything recovered": an epoch can have + // fenced some ids off as unrecovered already. Say which it was. + unrecovered, err := w.ledger.MissingIDs(ctx, loss.ID, LossUnrecovered) + if err != nil { + return false, err + } + if len(unrecovered) > 0 { + w.log.Warn("a buffer overflow's reconciliation ended with events unrecovered", "loss_id", loss.ID, "unrecovered", len(unrecovered)) + } else { + w.log.Info("a buffer overflow was fully reconciled", "loss_id", loss.ID) + } + return true, nil +} + +// walk runs one pass from the loss's own cursor to the frozen head, following +// `next` until it is absent. It returns the cursor to resume the next pass at. +func (w *repairWalker) walk(ctx context.Context, loss *Loss) (string, error) { + cursor := eventfeed.Cursor{Since: strconv.FormatInt(loss.RepairSince, 10)} + if loss.RepairCursor != "" { + cursor = eventfeed.Cursor{Position: loss.RepairCursor} + } + + // loss.RepairCursor is the walk's memory of what is DURABLE, and it is + // kept equal to what the ledger holds at every point below. A 410 is why: + // the epoch makes the pre-epoch position permanently unusable, and + // pollFailure clears it on disk — so a pass that then ends for any other + // reason must not hand a copy of that position back to its caller, which + // would put it back in memory and start every later pass at a position + // the feed has already refused. + pass := &repairPass{followed: map[string]bool{}} + walked := map[string]bool{} + maxPages := w.maxPages + if maxPages <= 0 { + maxPages = maxRepairPagesPerPass + } + for pages := 0; ; { + if err := ctx.Err(); err != nil { + // Cancellation is a delay, never a verdict: the loss stays open + // on disk for the next start. + return loss.RepairCursor, err + } + if pages >= maxPages { + // Distinct positions forever evade cycle detection. The pass ends + // at the cap, and the next one resumes from the cursor saved on + // the last page. + w.log.Warn("a repair pass reached its page cap; resuming on the repair cadence", "loss_id", loss.ID, "pages", pages) + return loss.RepairCursor, nil + } + pages++ + page, err := w.polls.Poll(ctx, cursor, w.filters) + if err != nil { + next, err := w.pollFailure(ctx, loss, cursor, err, pass) + if err != nil || next == nil { + // pollFailure has already written whatever this failure did + // to the durable cursor — cleared it at an epoch fence, or + // left it alone — so what is returned is that, never a copy + // of the position the failure just invalidated. + return loss.RepairCursor, err + } + cursor = *next + continue + } + + for _, event := range page.Events { + // Through intake like any other event: the ledger's dedupe is + // what marks the missing id recovered, and it is one code path + // rather than two that must agree. + if err := w.ingest(ctx, event, LaneRepair); err != nil { + return loss.RepairCursor, err + } + } + + if page.Position != "" { + // Saved first, remembered second: the walk's memory of the cursor + // never runs ahead of the ledger's. + if err := w.ledger.SaveRepairCursor(ctx, loss.ID, page.Position); err != nil { + return loss.RepairCursor, err + } + loss.RepairCursor = page.Position + } + + if page.Next == "" { + // The walk reached its frozen head. That is NOT "caught up": a + // page cut short by the safety horizon withholds the link on + // purpose, so the caller polls again rather than concluding + // anything. + return loss.RepairCursor, nil + } + // An empty page with a `next` is ordinary — the walk crossed rows the + // filters excluded — so the loop never stops on len(Events) == 0. + if walked[page.Next] || page.Next == cursor.PageURL { + // A next already walked this pass — itself, or a cycle — would + // spin. The pass ends and the repair cadence is the backoff. + w.log.Warn("a repair page's next repeats a URL this pass already walked; ending the pass", "loss_id", loss.ID) + return loss.RepairCursor, nil + } + walked[page.Next] = true + if err := sameOrigin(w.origin, page.Next); err != nil { + w.log.Error("a repair page's next URL leaves the API origin; the loss stays open for the next start", "loss_id", loss.ID) + return loss.RepairCursor, errReconciliationEnded + } + cursor = eventfeed.Cursor{PageURL: page.Next} + } +} + +// errReconciliationEnded reports that the walk itself settled the loss's fate +// for this start — closed it, or left it open for the next — so the caller +// stops rather than looping back to find nothing missing and calling that a +// full recovery. +var errReconciliationEnded = errors.New("connector: reconciliation ended inside the repair walk") + +// maxLossLifetime is how long a loss can stay open, however often a server +// asks for patience. A day is far past any horizon the feed itself has. +const maxLossLifetime = 24 * time.Hour + +// maxRepairPagesPerPass bounds the pages one repair pass walks. A thousand +// pages crosses up to a million ledger rows; past that the pass yields to the +// repair cadence and resumes from its saved cursor. +const maxRepairPagesPerPass = 1000 + +// maxResumesPerPass bounds the 410 resumes one pass follows. Keyed by URL +// alone, a server that signs or nonces its resume URLs would make every answer +// look new; the bound does not depend on the server choosing stable URLs. +const maxResumesPerPass = 2 + +// repairPass is what one pass has already done, stated rather than inferred +// from the shape of the cursor. +type repairPass struct { + // followed holds the resume URLs this pass has already followed. A 410 on + // one of them is the same 410 again. + followed map[string]bool + // reentered is whether this pass has already dropped a refused cursor + // for the explicit id. + reentered bool +} + +// failureKind is the only rendering of a failed poll the walk logs. A poll's +// error text can carry its request URL — a position token or a +// server-supplied continuation — and the walk logs at the operator's terminal. +func failureKind(err error) string { + var pollErr *eventfeed.PollError + if errors.As(err, &pollErr) { + return pollErr.Kind.String() + } + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return "canceled" + } + return "unclassified" +} + +// pollFailure decides what one failed repair poll means. It returns the cursor +// to continue this pass at, nil with no error to end the pass and wait for the +// next repair poll, or an error. +func (w *repairWalker) pollFailure(ctx context.Context, loss *Loss, cursor eventfeed.Cursor, err error, pass *repairPass) (*eventfeed.Cursor, error) { + if ctxErr := ctx.Err(); ctxErr != nil { + // The caller's cancellation, not the walk failing. It propagates, so + // nothing downstream mistakes a shutdown for a pass that ran. + return nil, ctxErr + } + var pollErr *eventfeed.PollError + if !errors.As(err, &pollErr) { + w.log.Warn("a repair poll failed; retrying on the repair cadence", "loss_id", loss.ID, "failure", failureKind(err)) + return nil, nil + } + + switch pollErr.Kind { + case eventfeed.PollGone: + epoch := pollErr.EpochAfterID + if pass.followed[pollErr.ResumeURL] || len(pass.followed) >= maxResumesPerPass { + // This pass already followed this resume, and it answered 410 + // again. Following it again would loop; the next pass tries. + w.log.Warn("the repair walk's resume answered 410 again; retrying on the repair cadence", "loss_id", loss.ID) + return nil, nil + } + if epoch != loss.RepairSince { + // A fence this loss has not seen: history up to the epoch is + // gone, and the ids there with it. + if _, recordErr := w.ledger.RecordGap(ctx, Gap{ + DetectedAt: w.now(), + Class: GapEpoch, + EpochAfterID: &epoch, + EntryClass: entryClassOf(pollErr.ResumeURL), + Note: "the repair walk was seeded below the feed's epoch", + }); recordErr != nil { + return nil, recordErr + } + behind, markErr := w.ledger.MarkUnrecoveredThrough(ctx, loss.ID, epoch) + if markErr != nil { + return nil, markErr + } + w.log.Error("the repair walk fell below the feed's epoch; the dropped events behind it are gone", + "loss_id", loss.ID, "epoch_after_id", epoch, "unrecovered", behind) + // Later passes start at the fence. + loss.RepairSince = epoch + loss.RepairCursor = "" + if setErr := w.ledger.SetRepairEntry(ctx, loss.ID, epoch); setErr != nil { + return nil, setErr + } + } + stillMissing, missErr := w.ledger.MissingIDs(ctx, loss.ID, LossMissing) + if missErr != nil { + return nil, missErr + } + if len(stillMissing) == 0 || pollErr.ResumeURL == "" { + if _, closeErr := w.ledger.CloseLoss(ctx, loss.ID, w.now()); closeErr != nil { + return nil, closeErr + } + return nil, errReconciliationEnded + } + // Ids above the epoch are still servable: the resume is followed as + // served, whether or not this loss had met the fence before. + if err := sameOrigin(w.origin, pollErr.ResumeURL); err != nil { + w.log.Error("a repair 410's resume URL leaves the API origin; the loss stays open for the next start", "loss_id", loss.ID) + return nil, errReconciliationEnded + } + pass.followed[pollErr.ResumeURL] = true + return &eventfeed.Cursor{PageURL: pollErr.ResumeURL}, nil + + case eventfeed.PollFilterChanged, eventfeed.PollPositionInvalid: + // The walk's own cursor was refused — minted under another filter + // set, or no longer honored. Its explicit id is still good. Once per + // pass: a refusal of the explicit id too is left to the cadence. + if pass.reentered || (cursor.Position == "" && cursor.PageURL == "") { + w.log.Warn("a repair poll was refused; retrying on the repair cadence", "loss_id", loss.ID, "failure", failureKind(err)) + return nil, nil + } + pass.reentered = true + loss.RepairCursor = "" + if saveErr := w.ledger.SaveRepairCursor(ctx, loss.ID, ""); saveErr != nil { + return nil, saveErr + } + return &eventfeed.Cursor{Since: strconv.FormatInt(loss.RepairSince, 10)}, nil + + case eventfeed.PollThrottled: + // The server named a wait. It is a directive, not a hint: polling + // again on the repair cadence would answer a fifteen-minute + // Retry-After every minute. + if pollErr.RetryAfter > w.retryAfter { + w.retryAfter = pollErr.RetryAfter + } + w.log.Warn("a repair poll was throttled; waiting as the server asked", + "loss_id", loss.ID, "retry_after", pollErr.RetryAfter) + return nil, nil + + case eventfeed.PollTransient, eventfeed.PollUnauthorized: + // A reason to try again on the next repair poll, not a reason to call + // the ids unrecovered. A slow or throttled walk delays nothing else. + w.log.Warn("a repair poll failed; retrying on the repair cadence", "loss_id", loss.ID, "failure", failureKind(err)) + return nil, nil + + case eventfeed.PollFilterInvalid, eventfeed.PollRedirectRefused, eventfeed.PollUnrecoverable: + } + + // Refused redirects, invalid filters, undifferentiated 400s, anything + // unrecoverable: no repeat inside this window will change the answer. + // The loss stays open, so the next start — perhaps after the cause is + // fixed — tries again instead of finding the ids already condemned. + w.log.Error("a repair poll failed in a way retrying will not fix; the loss stays open for the next start", + "loss_id", loss.ID, "failure", failureKind(err)) + return nil, errReconciliationEnded +} + +func (w *repairWalker) wait(ctx context.Context, d time.Duration) error { + if w.sleep != nil { + return w.sleep(ctx, d) + } + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-timer.C: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +// sameOrigin holds a URL the repair walk is about to follow to the API origin, +// with no scheme downgrade. +// +// This is the one check intake keeps from its temporary adapter. The SDK's +// live poll source re-issues a followed URL's cursor against its own client +// and never requests the URL itself, but it leaves origin validation to the +// caller — its connector runs it before every followed URL, unexported. The +// repair walk follows next and resume URLs outside that connector, so it +// runs the check itself rather than following a foreign URL's cursor. +func sameOrigin(origin, raw string) error { + parsed, err := url.Parse(raw) + if err != nil || !parsed.IsAbs() || parsed.User != nil { + return errForeignContinuation + } + canonical, err := eventfeed.CanonicalOrigin(parsed.Scheme + "://" + parsed.Host) + if err != nil || canonical != origin { + return errForeignContinuation + } + return nil +} + +var errForeignContinuation = errors.New("connector: a followed URL leaves the API origin") diff --git a/internal/connector/repair_bounds_test.go b/internal/connector/repair_bounds_test.go new file mode 100644 index 000000000..34958ef77 --- /dev/null +++ b/internal/connector/repair_bounds_test.go @@ -0,0 +1,251 @@ +package connector + +import ( + "context" + "errors" + "strconv" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" +) + +// nextPolls serves empty pages whose next URL is chosen by pick, and gives up +// after limit calls so a walk that never stops fails the test instead of +// hanging it. +type nextPolls struct { + mu sync.Mutex + calls int + limit int + pick func(call int) string +} + +func (n *nextPolls) Poll(context.Context, eventfeed.Cursor, eventfeed.Filters) (eventfeed.PollPage, error) { + n.mu.Lock() + defer n.mu.Unlock() + n.calls++ + if n.calls > n.limit { + return eventfeed.PollPage{}, &eventfeed.PollError{Kind: eventfeed.PollUnrecoverable} + } + return eventfeed.PollPage{Position: "pos-" + strconv.Itoa(n.calls), Next: n.pick(n.calls)}, nil +} + +const walkBase = "https://3.basecampapi.com/2914079/events.json?position=" + +func onePassWalker(t *testing.T, polls eventfeed.PollSource) (*repairWalker, *Ledger, Loss) { + t.Helper() + ledger := newTestLedger(t) + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + loss, err := ledger.RecordLoss(context.Background(), []int64{17099838509}, clock.at, 10*time.Minute, eventfeed.Filters{}) + require.NoError(t, err) + walker, _ := newTestWalker(t, ledger, polls, clock) + return walker, ledger, loss +} + +// C3: a next that points at itself ends the pass instead of spinning. +func TestARepairWalkStopsOnASelfReferentialNext(t *testing.T) { + polls := &nextPolls{limit: 300, pick: func(int) string { return walkBase + "same" }} + walker, _, loss := onePassWalker(t, polls) + + _, err := walker.walk(context.Background(), &loss) + require.NoError(t, err) + assert.LessOrEqual(t, polls.calls, 3, "a next already walked this pass is a cycle") +} + +// C3: an A→B→A cycle ends the pass too. +func TestARepairWalkStopsOnACycleOfNextURLs(t *testing.T) { + polls := &nextPolls{limit: 300, pick: func(call int) string { + if call%2 == 1 { + return walkBase + "A" + } + return walkBase + "B" + }} + walker, _, loss := onePassWalker(t, polls) + + _, err := walker.walk(context.Background(), &loss) + require.NoError(t, err) + assert.LessOrEqual(t, polls.calls, 4) +} + +// C3: distinct positions forever evade cycle detection, so a pass is also +// capped in pages, and returns the saved cursor so the next pass resumes. +func TestARepairPassIsCappedInPages(t *testing.T) { + polls := &nextPolls{limit: 300, pick: func(call int) string { return walkBase + "distinct-" + strconv.Itoa(call) }} + walker, ledger, loss := onePassWalker(t, polls) + walker.maxPages = 20 + + cursor, err := walker.walk(context.Background(), &loss) + require.NoError(t, err) + assert.Equal(t, 20, polls.calls) + assert.Equal(t, "pos-20", cursor, "the next pass resumes from the last saved page") + + open, err := ledger.OpenLosses(context.Background()) + require.NoError(t, err) + require.Len(t, open, 1) + assert.Equal(t, "pos-20", open[0].RepairCursor) +} + +// cancelingPolls cancels the walk's context on its first call, as a shutdown +// arriving mid-walk would, and answers with the context's error. +type cancelingPolls struct { + cancel context.CancelFunc + calls int +} + +func (c *cancelingPolls) Poll(ctx context.Context, _ eventfeed.Cursor, _ eventfeed.Filters) (eventfeed.PollPage, error) { + c.calls++ + c.cancel() + return eventfeed.PollPage{}, ctx.Err() +} + +// C4: cancellation is not the walk failing. A shutdown mid-walk leaves the +// loss open, with nothing condemned, and the next start completes it. +func TestACanceledRepairWalkLeavesTheLossOpenForTheNextStart(t *testing.T) { + for name, age := range map[string]time.Duration{ + "inside the window": 0, + // Past its window, but well inside the absolute deadline: this is + // about cancellation, not about a loss that has run out of time. + "on the final pass": time.Hour, + } { + t.Run(name, func(t *testing.T) { + ledger := newTestLedger(t) + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + loss, err := ledger.RecordLoss(context.Background(), []int64{17099838509}, clock.at.Add(-age), 10*time.Minute, eventfeed.Filters{}) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + walker, _ := newTestWalker(t, ledger, &cancelingPolls{cancel: cancel}, clock) + err = walker.reconcile(ctx, loss) + assert.ErrorIs(t, err, context.Canceled) + + open, err := ledger.OpenLosses(context.Background()) + require.NoError(t, err) + require.Len(t, open, 1, "a shutdown mid-walk is a delay, not a verdict") + unrecovered, err := ledger.UnrecoveredIDs(context.Background()) + require.NoError(t, err) + assert.Empty(t, unrecovered) + + // The next start completes it. + restart, _ := newTestWalker(t, ledger, &scriptedPolls{pages: []eventfeed.PollPage{ + {Events: []eventfeed.Event{testEvent(17099838509)}, Position: "after"}, + }}, clock) + require.NoError(t, restart.reconcile(context.Background(), open[0])) + open, err = ledger.OpenLosses(context.Background()) + require.NoError(t, err) + assert.Empty(t, open) + recovered, err := ledger.MissingIDs(context.Background(), loss.ID, LossRecovered) + require.NoError(t, err) + assert.Equal(t, []int64{17099838509}, recovered) + }) + } +} + +// C3: a throttled repair poll waits at least as long as the server said. The +// seam's RetryAfter is a directive, not a hint: polling again on the repair +// cadence would answer a fifteen-minute wait every minute. +func TestARepairWalkHonoursRetryAfter(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, clock.at, time.Hour, eventfeed.Filters{}) + require.NoError(t, err) + + polls := &scriptedPolls{ + errs: []error{&eventfeed.PollError{Kind: eventfeed.PollThrottled, RetryAfter: 15 * time.Minute}}, + pages: []eventfeed.PollPage{{}, {Events: []eventfeed.Event{testEvent(17099838509)}, Position: "p"}}, + } + walker, _ := newTestWalker(t, ledger, polls, clock) + var waits []time.Duration + walker.sleep = func(_ context.Context, d time.Duration) error { + waits = append(waits, d) + clock.at = clock.at.Add(d) + return nil + } + require.NoError(t, walker.reconcile(ctx, loss)) + + require.NotEmpty(t, waits) + assert.GreaterOrEqual(t, waits[0], 15*time.Minute, + "the server asked for fifteen minutes; the repair cadence does not overrule it") +} + +// C2: a throttle is not a verdict. A server that only ever asks for patience +// can never turn a loss into unrecovered ids, however long its window has been +// over: the wait is the server's time, not the loss's. +func TestAPermanentlyThrottlingServerNeverCondemnsALoss(t *testing.T) { + ledger := newTestLedger(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, clock.at.Add(-time.Hour), time.Minute, eventfeed.Filters{}) + require.NoError(t, err) + + polls := &alwaysThrottling{} + walker, _ := newTestWalker(t, ledger, polls, clock) + waits := 0 + walker.sleep = func(_ context.Context, d time.Duration) error { + clock.at = clock.at.Add(d) + if waits++; waits >= 5 { + cancel() + return ctx.Err() + } + return nil + } + require.ErrorIs(t, walker.reconcile(ctx, loss), context.Canceled) + + open, err := ledger.OpenLosses(context.Background()) + require.NoError(t, err) + require.Len(t, open, 1, "still scheduled, never condemned") + assert.True(t, open[0].DeadlineAt.After(loss.DeadlineAt), "the window moved out by what the server asked for") + unrecovered, err := ledger.UnrecoveredIDs(context.Background()) + require.NoError(t, err) + assert.Empty(t, unrecovered) +} + +type alwaysThrottling struct{} + +func (alwaysThrottling) Poll(context.Context, eventfeed.Cursor, eventfeed.Filters) (eventfeed.PollPage, error) { + return eventfeed.PollPage{}, &eventfeed.PollError{Kind: eventfeed.PollThrottled, RetryAfter: 15 * time.Minute} +} + +// A throttle is not a failure, and the wait it asks for is the server's to +// name — but the loss's absolute deadline is this connector's promise, and an +// uncapped Retry-After would hold the loss open, and its repair worker with +// it, well past the day it is allowed to live. +func TestAThrottleCannotHoldALossPastItsAbsoluteDeadline(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + detected := time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC) + clock := &walkClock{at: detected} + loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, detected, time.Hour, eventfeed.Filters{}) + require.NoError(t, err) + + // Twenty-three hours in, the server asks for six more. + clock.at = detected.Add(23 * time.Hour) + var slept []time.Duration + polls := &scriptedPolls{errs: []error{ + &eventfeed.PollError{Kind: eventfeed.PollThrottled, RetryAfter: 6 * time.Hour, Err: errors.New("slow down")}, + }} + walker, _ := newTestWalker(t, ledger, polls, clock) + walker.sleep = func(_ context.Context, d time.Duration) error { + slept = append(slept, d) + clock.at = clock.at.Add(d) + return nil + } + + require.NoError(t, walker.reconcile(ctx, loss)) + + require.NotEmpty(t, slept) + for _, d := range slept { + assert.LessOrEqual(t, d, time.Hour, "no wait may reach past the loss's last hour") + } + assert.LessOrEqual(t, clock.at.Sub(detected), maxLossLifetime, "the loss outlived its deadline") + + open, err := ledger.OpenLosses(ctx) + require.NoError(t, err) + assert.Empty(t, open, "the loss closes at its deadline whatever the server asked for") +} diff --git a/internal/connector/repair_test.go b/internal/connector/repair_test.go new file mode 100644 index 000000000..b530eb4f4 --- /dev/null +++ b/internal/connector/repair_test.go @@ -0,0 +1,269 @@ +package connector + +import ( + "context" + "log/slog" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" +) + +type walkClock struct{ at time.Time } + +func (c *walkClock) now() time.Time { return c.at } +func (c *walkClock) advance(d time.Duration) func(context.Context, time.Duration) error { + return func(context.Context, time.Duration) error { + c.at = c.at.Add(d) + return nil + } +} + +func newTestWalker(t *testing.T, ledger *Ledger, polls eventfeed.PollSource, clock *walkClock) (*repairWalker, *[]int64) { + t.Helper() + var ingested []int64 + walker := &repairWalker{ + ledger: ledger, + polls: polls, + origin: "https://3.basecampapi.com", + now: clock.now, + log: slog.New(slog.DiscardHandler), + sleep: clock.advance(time.Minute), + ingest: func(ctx context.Context, event eventfeed.Event, lane Lane) error { + ingested = append(ingested, event.ID) + _, err := ledger.RecordSeen(ctx, event, lane) + return err + }, + interval: time.Minute, + } + return walker, &ingested +} + +func TestRepairWalkEntersOneBelowTheLowestMissingID(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + + loss, err := ledger.RecordLoss(ctx, []int64{17099838505, 17099838502}, clock.at, 10*time.Minute, eventfeed.Filters{}) + require.NoError(t, err) + + polls := &scriptedPolls{pages: []eventfeed.PollPage{{ + Events: []eventfeed.Event{testEvent(17099838502), testEvent(17099838505)}, + Position: "walk-1", + }}} + walker, ingested := newTestWalker(t, ledger, polls, clock) + require.NoError(t, walker.reconcile(ctx, loss)) + + require.NotEmpty(t, polls.cursors) + assert.Equal(t, "17099838501", polls.cursors[0].Since, + "the feed's since is exclusive, so the walk enters one below the lowest missing id") + assert.Equal(t, []int64{17099838502, 17099838505}, *ingested) + + recovered, err := ledger.MissingIDs(ctx, loss.ID, LossRecovered) + require.NoError(t, err) + assert.Equal(t, []int64{17099838502, 17099838505}, recovered) +} + +// A request crosses up to a thousand ledger rows and serves at most a hundred +// matches; the rows the filters excluded still advanced the cursor. Stopping at +// the first empty page abandons the repair one page short. +func TestRepairWalkFollowsNextThroughAnEmptyPage(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + + loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, clock.at, 10*time.Minute, eventfeed.Filters{}) + require.NoError(t, err) + + polls := &scriptedPolls{pages: []eventfeed.PollPage{ + {Events: nil, Position: "walk-1", Next: "https://3.basecampapi.com/2914079/events.json?position=walk-1"}, + {Events: []eventfeed.Event{testEvent(17099838509)}, Position: "walk-2"}, + }} + walker, ingested := newTestWalker(t, ledger, polls, clock) + require.NoError(t, walker.reconcile(ctx, loss)) + + assert.Equal(t, 2, polls.calls) + assert.Equal(t, []int64{17099838509}, *ingested) + assert.Equal(t, "https://3.basecampapi.com/2914079/events.json?position=walk-1", polls.cursors[1].PageURL) +} + +// A missing `next` means the walk reached its frozen head, not that history +// ended: a page cut short by the safety horizon withholds the link on purpose. +func TestRepairWalkRepeatsAfterAMissingNext(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + + loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, clock.at, 10*time.Minute, eventfeed.Filters{}) + require.NoError(t, err) + + polls := &scriptedPolls{pages: []eventfeed.PollPage{ + {Events: nil, Position: "walk-1"}, + {Events: nil, Position: "walk-2"}, + {Events: []eventfeed.Event{testEvent(17099838509)}, Position: "walk-3"}, + }} + walker, ingested := newTestWalker(t, ledger, polls, clock) + require.NoError(t, walker.reconcile(ctx, loss)) + + assert.Equal(t, 3, polls.calls, "the walk repeats on the repair cadence rather than concluding") + assert.Equal(t, []int64{17099838509}, *ingested) + assert.Equal(t, "walk-2", polls.cursors[2].Position, + "each repeat resumes from the walk's own cursor") +} + +// The walk is seeded from a live id that ran far ahead of the poll lane. Its +// cursor belongs to the loss record and must never reach the feed's checkpoint. +func TestRepairCursorNeverReachesTheFeedCheckpoint(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + key := testKey() + require.NoError(t, ledger.Save(ctx, key, "feed-position-before-the-overflow")) + + loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, clock.at, 10*time.Minute, eventfeed.Filters{}) + require.NoError(t, err) + + polls := &scriptedPolls{pages: []eventfeed.PollPage{{ + Events: []eventfeed.Event{testEvent(17099838509)}, + Position: "repair-walk-position", + }}} + walker, _ := newTestWalker(t, ledger, polls, clock) + require.NoError(t, walker.reconcile(ctx, loss)) + + position, ok, err := ledger.Load(ctx, key) + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, "feed-position-before-the-overflow", position, + "a checkpoint taken from the repair walk would skip everything inside the safety delay behind it") +} + +func TestRepairWalkSavesItsOwnCursorOnTheLoss(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + + loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, clock.at, 10*time.Minute, eventfeed.Filters{}) + require.NoError(t, err) + + polls := &scriptedPolls{pages: []eventfeed.PollPage{{Events: nil, Position: "repair-walk-position"}}} + walker, _ := newTestWalker(t, ledger, polls, clock) + // One pass only: stop the clock past the deadline so the walk gives up. + walker.sleep = func(context.Context, time.Duration) error { + clock.at = clock.at.Add(11 * time.Minute) + return nil + } + require.NoError(t, walker.reconcile(ctx, loss)) + + open, err := ledger.OpenLosses(ctx) + require.NoError(t, err) + assert.Empty(t, open) + + unrecovered, err := ledger.UnrecoveredIDs(ctx) + require.NoError(t, err) + assert.Equal(t, []int64{17099838509}, unrecovered) +} + +func TestIdsStillMissingWhenTheWindowClosesAreUnrecovered(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + + loss, err := ledger.RecordLoss(ctx, []int64{17099838509, 17099838510}, clock.at, 10*time.Minute, eventfeed.Filters{}) + require.NoError(t, err) + + // The straggler arrives on the fourth repair poll; the other never does. + polls := &scriptedPolls{pages: []eventfeed.PollPage{ + {Position: "w1"}, + {Position: "w2"}, + {Position: "w3"}, + {Events: []eventfeed.Event{testEvent(17099838510)}, Position: "w4"}, + }} + walker, _ := newTestWalker(t, ledger, polls, clock) + require.NoError(t, walker.reconcile(ctx, loss)) + + assert.GreaterOrEqual(t, polls.calls, 10, "ten minutes at a sixty-second cadence") + + recovered, err := ledger.MissingIDs(ctx, loss.ID, LossRecovered) + require.NoError(t, err) + assert.Equal(t, []int64{17099838510}, recovered) + + unrecovered, err := ledger.UnrecoveredIDs(ctx) + require.NoError(t, err) + assert.Equal(t, []int64{17099838509}, unrecovered, + "a late straggler is reported, never hidden") +} + +// An unrecovered id the poll lane serves later is resolved by intake like any +// other event — the ledger is one dedupe, not two. +func TestAnUnrecoveredIDIsStillIngestedNormallyLater(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + + loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, time.Now(), time.Minute, eventfeed.Filters{}) + require.NoError(t, err) + _, err = ledger.CloseLoss(ctx, loss.ID, time.Now()) + require.NoError(t, err) + + fresh, err := ledger.RecordSeen(ctx, testEvent(17099838509), LanePoll) + require.NoError(t, err) + assert.True(t, fresh) + + record, ok, err := ledger.Get(ctx, 17099838509) + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, StateSeen, record.State) +} + +func TestRepairWalkBelowTheEpochEndsWithAGap(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + + loss, err := ledger.RecordLoss(ctx, []int64{100, 101}, clock.at, 10*time.Minute, eventfeed.Filters{}) + require.NoError(t, err) + + polls := &scriptedPolls{errs: []error{&eventfeed.PollError{ + Kind: eventfeed.PollGone, + EpochAfterID: 17099838487, + ResumeURL: "https://3.basecampapi.com/2914079/events.json?since=17099838487", + }}} + walker, _ := newTestWalker(t, ledger, polls, clock) + require.NoError(t, walker.reconcile(ctx, loss)) + + assert.Equal(t, 1, polls.calls, "no number of repeats will serve history below the epoch") + + gaps, err := ledger.Gaps(ctx) + require.NoError(t, err) + require.Len(t, gaps, 1) + assert.Equal(t, GapEpoch, gaps[0].Class) + + unrecovered, err := ledger.UnrecoveredIDs(ctx) + require.NoError(t, err) + assert.Equal(t, []int64{100, 101}, unrecovered) +} + +func TestATransientRepairPollIsRetriedNotGivenUpOn(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + + loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, clock.at, 10*time.Minute, eventfeed.Filters{}) + require.NoError(t, err) + + polls := &scriptedPolls{ + errs: []error{&eventfeed.PollError{Kind: eventfeed.PollThrottled, RetryAfter: time.Second}, nil}, + pages: []eventfeed.PollPage{{}, {Events: []eventfeed.Event{testEvent(17099838509)}, Position: "w2"}}, + } + walker, ingested := newTestWalker(t, ledger, polls, clock) + require.NoError(t, walker.reconcile(ctx, loss)) + + assert.Equal(t, []int64{17099838509}, *ingested, + "a throttled walk delays the repair; it does not condemn the ids") + + unrecovered, err := ledger.UnrecoveredIDs(ctx) + require.NoError(t, err) + assert.Empty(t, unrecovered) +} diff --git a/internal/connector/requeue_test.go b/internal/connector/requeue_test.go new file mode 100644 index 000000000..0996b9fb2 --- /dev/null +++ b/internal/connector/requeue_test.go @@ -0,0 +1,189 @@ +package connector + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type failingWriter struct{} + +func (failingWriter) Write([]byte) (int, error) { return 0, errors.New("stdout is gone") } + +// B2: the ledger row is written before the pointer line and the hand-off, so +// that a crash cannot lose the pointer. The cost is that a failure after the +// commit leaves an event the ledger's dedupe will suppress on every retry. +// Nothing may be left in that state: a record still seen is queued again on +// the next start. +func TestAnEventWhoseHandOffFailedIsQueuedOnTheNextStart(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + + // A first run whose stdout is gone: the row commits, the pointer line + // fails, and the id never reaches the queue. + first, _, firstQueue := newTestIntakeOn(t, ledger, failingWriter{}) + require.Error(t, first.ingest(ctx, testEvent(17099838500), LanePoll)) + require.Zero(t, firstQueue.Depth()) + + record, ok, err := ledger.Get(ctx, 17099838500) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, StateSeen, record.State) + + // The next start hands it over. + second, _, secondQueue := newTestIntakeOn(t, ledger, nil) + require.NoError(t, second.requeueSeen(ctx)) + assert.Equal(t, 1, secondQueue.Depth()) + id, err := secondQueue.Take(ctx) + require.NoError(t, err) + assert.Equal(t, int64(17099838500), id) +} + +// Records a later stage has already taken are not queued again. +func TestOnlyRecordsStillSeenAreQueuedOnAStart(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + for _, id := range []int64{1, 2, 3} { + _, err := ledger.RecordSeen(ctx, testEvent(id), LanePoll) + require.NoError(t, err) + } + require.NoError(t, ledger.SetState(ctx, 2, StateAdmitted, "")) + require.NoError(t, ledger.SetState(ctx, 3, StateDiscarded, "untrusted_author")) + + intake, _, queue := newTestIntakeOn(t, ledger, nil) + require.NoError(t, intake.requeueSeen(ctx)) + assert.Equal(t, 1, queue.Depth()) + id, err := queue.Take(ctx) + require.NoError(t, err) + assert.Equal(t, int64(1), id) +} + +// The hand-off waits for room rather than dropping, and a shutdown mid-requeue +// leaves the rest for the next start. +func TestRequeueingStopsOnShutdownAndLeavesTheRest(t *testing.T) { + ledger := newTestLedger(t) + ctx, cancel := context.WithCancel(context.Background()) + for _, id := range []int64{1, 2, 3} { + _, err := ledger.RecordSeen(ctx, testEvent(id), LanePoll) + require.NoError(t, err) + } + intake, _, queue := newTestIntakeOn(t, ledger, nil) + small, err := NewQueue(1, 1) + require.NoError(t, err) + intake.queue = small + _ = queue + + done := make(chan error, 1) + go func() { done <- intake.requeueSeen(ctx) }() + require.Eventually(t, small.Paused, 2*time.Second, time.Millisecond) + cancel() + select { + case err := <-done: + assert.ErrorIs(t, err, context.Canceled) + case <-time.After(2 * time.Second): + t.Fatal("requeueing should stop on shutdown") + } + + remaining, err := ledger.RecordsInState(context.Background(), StateSeen, 10) + require.NoError(t, err) + assert.Len(t, remaining, 3, "nothing is consumed by being queued; the next start sees them all") +} + +// The restart is not the only answer. A repair walk is the case that makes it +// urgent: the commit resolves the loss's missing id, so the reconciliation +// that follows sees nothing missing and closes, and a connector that runs for +// weeks would never mention the event again. Every commit goes through one +// handover path, and a failure on it is remembered and retried in process. +func TestAnEventWhoseHandOffFailedIsSweptWithoutARestart(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + intake, _, queue := newTestIntakeOn(t, ledger, failingWriter{}) + + require.Error(t, intake.ingest(ctx, testEvent(17099838500), LaneRepair)) + require.Zero(t, queue.Depth(), "the id never reached the queue") + + // The ledger's dedupe now suppresses the event on every later delivery of + // itself, so nothing but the sweep can hand it over. + fresh, err := ledger.RecordSeen(ctx, testEvent(17099838500), LaneRepair) + require.NoError(t, err) + require.False(t, fresh) + + intake.sweepStranded(ctx) + + require.Equal(t, 1, queue.Depth()) + id, err := queue.Take(ctx) + require.NoError(t, err) + assert.Equal(t, int64(17099838500), id) + + // Exactly once: a second sweep has nothing left to offer. + intake.sweepStranded(ctx) + assert.Zero(t, queue.Depth()) +} + +// A sweep that cannot hand an id over leaves it for the next one, and for the +// next start after that. +func TestASweepThatCannotHandOverKeepsTheStrandedID(t *testing.T) { + ctx := context.Background() + intake, _, _ := newTestIntakeOn(t, newTestLedger(t), failingWriter{}) + full, err := NewQueue(1, 1) + require.NoError(t, err) + intake.queue = full + + require.Error(t, intake.ingest(ctx, testEvent(42), LaneLive)) + require.NoError(t, full.Offer(ctx, 99)) // no room for anything else + + stopped, cancel := context.WithCancel(ctx) + cancel() + intake.sweepStranded(stopped) + assert.Equal(t, 1, full.Depth(), "the sweep waited for room and gave up, keeping the id") + + waiting, err := full.Take(ctx) + require.NoError(t, err) + require.Equal(t, int64(99), waiting) + + intake.sweepStranded(ctx) + bounded, stop := context.WithTimeout(ctx, 5*time.Second) + defer stop() + id, err := full.Take(bounded) + require.NoError(t, err, "the id the earlier sweep kept should be handed over now") + assert.Equal(t, int64(42), id) +} + +// The start's re-queue offers what is in seen and then forgets the ids it +// carried in — but only those. A repair walk is already running by then, and +// it serves OLD ids, which the paging may already have passed: one stranded +// mid-pass must not be forgotten by a clear that assumed it had offered +// everything. +func TestTheStartOnlyForgetsTheStrandedIDsItCarriedIn(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + _, err := ledger.RecordSeen(ctx, testEvent(1), LanePoll) + require.NoError(t, err) + + intake, _, _ := newTestIntakeOn(t, ledger, nil) + queue, err := NewQueue(1, 10) + require.NoError(t, err) + intake.queue = queue + var once sync.Once + // Stranded while the re-queue is paging: a repair walk's hand-off failing + // on an id the paging has already gone past. + queue.OnWarn = func(int) { once.Do(func() { intake.strand(999) }) } + + require.NoError(t, intake.requeueSeen(ctx)) + require.Equal(t, 1, queue.Depth()) + + intake.sweepStranded(ctx) + + require.Equal(t, 2, queue.Depth(), "the id stranded mid-pass was never offered") + first, err := queue.Take(ctx) + require.NoError(t, err) + assert.Equal(t, int64(1), first) + second, err := queue.Take(ctx) + require.NoError(t, err) + assert.Equal(t, int64(999), second) +} diff --git a/internal/connector/review2_test.go b/internal/connector/review2_test.go new file mode 100644 index 000000000..619e76d6d --- /dev/null +++ b/internal/connector/review2_test.go @@ -0,0 +1,304 @@ +package connector + +import ( + "context" + "os" + "path/filepath" + "runtime" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" +) + +// Round 2 of review. Each of these failed on 893c0d6. + +func TestARefusedPositionPrefersThisFilterSetsOwnPollServedID(t *testing.T) { + ledger := newTestLedger(t) + intake, transport, minter, polls := newFeedIntake(t, ledger, Options{}) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + key := intake.CheckpointKey() + require.NoError(t, ledger.Save(ctx, key, "position-the-server-will-refuse")) + require.NoError(t, ledger.NotePollServed(ctx, key, 1000)) + other := key + other.FilterKey = "srv2-0000000000000000" + require.NoError(t, ledger.NotePollServed(ctx, other, 2000)) + + minter.ScriptTicket(ticket()) + minter.ScriptTicket(ticket()) + polls.ScriptError(&eventfeed.PollError{Kind: eventfeed.PollPositionInvalid}) + polls.ScriptPage(eventfeed.PollPage{Position: "p2"}) + polls.ScriptPage(eventfeed.PollPage{Position: "p3"}) + + done := runInBackground(ctx, t, intake) + subscribedConn(t, transport) + + answered := 1 + var reentry eventfeed.Cursor + require.Eventually(t, func() bool { + if conns := transport.Conns(); len(conns) > answered { + answerSubscription(conns[len(conns)-1]) + answered = len(conns) + } + calls := polls.Calls() + if len(calls) < 2 { + return false + } + reentry = calls[1].Cursor + return true + }, 5*time.Second, 10*time.Millisecond) + assert.Equal(t, "1000", reentry.Since, + "another filter set's id is past events this one never served; re-entering there skips them") + + cancel() + awaitReturn(t, done, "Run should return on shutdown") +} + +// A reconnect that lands while intake is waiting for queue room must not +// strand the event it was handing over: it is already seen, so no re-served +// page would ever queue it again. +func TestAReconnectDuringABackloggedHandOffStrandsNothing(t *testing.T) { + ledger := newTestLedger(t) + queue, err := NewQueue(1, 1) + require.NoError(t, err) + membership := &flakyMembership{buckets: []int64{48699913}} + intake, transport, minter, polls := newFeedIntake(t, ledger, Options{ + Membership: membership, + MembershipInterval: 50 * time.Millisecond, + }) + intake.queue = queue + intake.opts.Queue = queue + for range 5 { + minter.ScriptTicket(ticket()) + } + page := eventfeed.PollPage{Events: []eventfeed.Event{testEvent(500), testEvent(501)}, Position: "p1"} + for range 5 { + polls.ScriptPage(page) + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := runInBackground(ctx, t, intake) + subscribedConn(t, transport) + + require.Eventually(t, func() bool { return queue.Paused() }, 5*time.Second, 5*time.Millisecond) + membership.mu.Lock() + membership.buckets = []int64{48699913, 1} + membership.mu.Unlock() + + // Nothing is taken until the reconnect has been asked for, so the hand-off + // is still waiting when it lands. + require.Eventually(t, func() bool { + return len(transport.Dials()) >= 2 || len(intake.reconnect) == 1 + }, 5*time.Second, time.Millisecond) + time.Sleep(20 * time.Millisecond) + + answered := 1 + var got []int64 + require.Eventually(t, func() bool { + if conns := transport.Conns(); len(conns) > answered { + answerSubscription(conns[len(conns)-1]) + answered = len(conns) + } + ctxTake, stop := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer stop() + if id, err := queue.Take(ctxTake); err == nil { + got = append(got, id) + } + return len(got) >= 2 + }, 5*time.Second, 10*time.Millisecond, "every seen event must reach the queue") + assert.Equal(t, []int64{500, 501}, got) + + cancel() + awaitReturn(t, done, "Run should return on shutdown") +} + +// A project the membership list never names — archived, or past the lister's +// page — must cost one reconnect, not one per event. +func TestAnUnlistedProjectCostsOneReconnectNotOnePerEvent(t *testing.T) { + ledger := newTestLedger(t) + intake, transport, minter, polls := newFeedIntake(t, ledger, Options{ + Membership: &flakyMembership{buckets: []int64{1}}, + MembershipInterval: time.Hour, + }) + for range 20 { + minter.ScriptTicket(ticket()) + } + var events []eventfeed.Event + for id := int64(600); id < 605; id++ { + e := testEvent(id) + e.BucketID = 777 + events = append(events, e) + } + for range 20 { + polls.ScriptPage(eventfeed.PollPage{Events: events, Position: "p"}) + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := runInBackground(ctx, t, intake) + subscribedConn(t, transport) + + answered := 1 + deadline := time.Now().Add(time.Second) + for time.Now().Before(deadline) { + if conns := transport.Conns(); len(conns) > answered { + answerSubscription(conns[len(conns)-1]) + answered = len(conns) + } + time.Sleep(10 * time.Millisecond) + } + assert.LessOrEqual(t, len(transport.Dials()), 2, "five events from one unlisted project are one reconnect") + + cancel() + awaitReturn(t, done, "Run should return on shutdown") +} + +func TestSinceAppliesToTheFirstConnectionOnly(t *testing.T) { + ledger := newTestLedger(t) + intake, transport, minter, polls := newFeedIntake(t, ledger, Options{ + SinceEventID: 17099838000, + Membership: &flakyMembership{buckets: []int64{1}}, + MembershipInterval: time.Hour, + }) + minter.ScriptTicket(ticket()) + minter.ScriptTicket(ticket()) + e := testEvent(17099838001) + e.BucketID = 777 // unlisted: forces one reconnect + polls.ScriptPage(eventfeed.PollPage{Events: []eventfeed.Event{e}, Position: "after-since"}) + polls.ScriptPage(eventfeed.PollPage{Position: "second"}) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := runInBackground(ctx, t, intake) + subscribedConn(t, transport) + + answered := 1 + require.Eventually(t, func() bool { + if conns := transport.Conns(); len(conns) > answered { + answerSubscription(conns[len(conns)-1]) + answered = len(conns) + } + return polls.CallCount() >= 2 + }, 5*time.Second, 10*time.Millisecond) + assert.Equal(t, "17099838000", polls.Calls()[0].Cursor.Since) + assert.NotEqual(t, "17099838000", polls.Calls()[1].Cursor.Since, + "a reconnect resumes from what the run stored, not from --since again") + + cancel() + awaitReturn(t, done, "Run should return on shutdown") +} + +func TestMembershipWatcherStopsWhenStopped(t *testing.T) { + var reads atomic.Int32 + intake, _, _ := newTestIntake(t, nil, nil) + intake.opts.Membership = countingMembership{&reads} + intake.opts.MembershipInterval = time.Millisecond + + ctx, stop := context.WithCancel(context.Background()) + defer stop() + await := intake.watchMembership(ctx) + require.Eventually(t, func() bool { return reads.Load() > 0 }, time.Second, time.Millisecond) + stop() + await() + after := reads.Load() + time.Sleep(20 * time.Millisecond) + assert.Equal(t, after, reads.Load(), "a stopped watcher reads nothing more") +} + +type countingMembership struct{ n *atomic.Int32 } + +func (c countingMembership) Buckets(context.Context) ([]int64, error) { + c.n.Add(1) + return nil, nil +} + +func TestQueuePauseTracksEveryBlockedOffer(t *testing.T) { + queue, err := NewQueue(1, 1) + require.NoError(t, err) + var pauses, resumes atomic.Int32 + queue.OnPause = func(int) { pauses.Add(1) } + queue.OnResume = func(int) { resumes.Add(1) } + ctx := context.Background() + require.NoError(t, queue.Offer(ctx, 1)) + + first := make(chan error, 1) + second := make(chan error, 1) + go func() { first <- queue.Offer(ctx, 2) }() + go func() { second <- queue.Offer(ctx, 3) }() + require.Eventually(t, func() bool { return pauses.Load() >= 1 && len(queue.ids) == 1 }, time.Second, time.Millisecond) + time.Sleep(20 * time.Millisecond) + + _, err = queue.Take(ctx) + require.NoError(t, err) + select { + case <-first: + case <-second: + case <-time.After(time.Second): + t.Fatal("one offer should have resumed") + } + assert.True(t, queue.Paused(), "the other offer is still waiting, so the feed is still paused") + assert.Zero(t, resumes.Load(), "resume fires when the last waiter stops waiting") + + _, err = queue.Take(ctx) + require.NoError(t, err) + require.Eventually(t, func() bool { return !queue.Paused() }, time.Second, time.Millisecond) + assert.Equal(t, int32(1), pauses.Load()) + assert.Equal(t, int32(1), resumes.Load()) +} + +// The ledger holds feed positions — signed tokens — and account event +// metadata. It is private to the user or it is not opened. +func TestLedgerIsCreatedPrivate(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("POSIX permission bits") + } + dir := filepath.Join(t.TempDir(), "state") + ledger, err := OpenLedger(filepath.Join(dir, "connector.db")) + require.NoError(t, err) + defer ledger.Close() + require.NoError(t, ledger.Save(context.Background(), testKey(), "p")) + + info, err := os.Stat(dir) + require.NoError(t, err) + assert.Equal(t, os.FileMode(0o700), info.Mode().Perm()) + for _, name := range []string{"connector.db", "connector.db-wal", "connector.db-shm"} { + info, err := os.Stat(filepath.Join(dir, name)) + if os.IsNotExist(err) { + continue + } + require.NoError(t, err) + assert.Equal(t, os.FileMode(0o600), info.Mode().Perm(), name) + } +} + +func TestLedgerRefusesALooseDirectory(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("POSIX permission bits") + } + dir := filepath.Join(t.TempDir(), "shared") + require.NoError(t, os.Mkdir(dir, 0o755)) + require.NoError(t, os.Chmod(dir, 0o755)) + _, err := OpenLedger(filepath.Join(dir, "connector.db")) + assert.Error(t, err, "a directory other users can read exposes the ledger's sidecars") +} + +func TestLedgerRefusesALooseFile(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("POSIX permission bits") + } + dir := filepath.Join(t.TempDir(), "state") + require.NoError(t, os.Mkdir(dir, 0o700)) + require.NoError(t, os.Chmod(dir, 0o700)) + path := filepath.Join(dir, "connector.db") + require.NoError(t, os.WriteFile(path, nil, 0o644)) + require.NoError(t, os.Chmod(path, 0o644)) + _, err := OpenLedger(path) + assert.Error(t, err) +} diff --git a/internal/connector/review_fixes_test.go b/internal/connector/review_fixes_test.go new file mode 100644 index 000000000..a3b89bcf3 --- /dev/null +++ b/internal/connector/review_fixes_test.go @@ -0,0 +1,420 @@ +package connector + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "net/http" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed/feedtest" +) + +func TestALateArrivalClearsAnUnrecoveredID(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + + loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, time.Now(), time.Minute, eventfeed.Filters{}) + require.NoError(t, err) + _, err = ledger.CloseLoss(ctx, loss.ID, time.Now()) + require.NoError(t, err) + + _, err = ledger.RecordSeen(ctx, testEvent(17099838509), LanePoll) + require.NoError(t, err) + + unrecovered, err := ledger.UnrecoveredIDs(ctx) + require.NoError(t, err) + assert.Empty(t, unrecovered, "an id intake received is not unrecovered, however late it came") +} + +func TestPointerLineReportsTheLaneIntakeWasGiven(t *testing.T) { + var pointers bytes.Buffer + intake, _, _ := newTestIntake(t, nil, &pointers) + require.NoError(t, intake.ingest(context.Background(), testEvent(1), LaneRepair)) + + var pointer Pointer + require.NoError(t, json.Unmarshal([]byte(strings.TrimSpace(pointers.String())), &pointer)) + assert.Equal(t, LaneRepair, pointer.Lane, "the pointer line and the ledger must agree on the lane") +} + +func TestMembershipBaselineIsTakenFromTheFirstSuccessfulRead(t *testing.T) { + intake, _, _ := newTestIntake(t, nil, nil) + + // The snapshot at subscribe failed, so there is none. + assert.False(t, intake.membershipChanged([]int64{1, 2}), "the first successful read is a baseline, not a change") + assert.True(t, intake.membershipChanged([]int64{1, 2, 3}), "and a later grant is then seen") +} + +type flakyMembership struct { + mu sync.Mutex + buckets []int64 + err error +} + +func (m *flakyMembership) Buckets(context.Context) ([]int64, error) { + m.mu.Lock() + defer m.mu.Unlock() + return m.buckets, m.err +} + +func newFeedIntake(t *testing.T, ledger *Ledger, opts Options) (*Intake, *feedtest.Transport, *feedtest.Minter, *feedtest.Polls) { + t.Helper() + queue, err := NewQueue(DefaultBacklogWarn, DefaultBacklogPause) + require.NoError(t, err) + transport := feedtest.NewTransport() + minter := feedtest.NewMinter() + polls := feedtest.NewPolls() + opts.Origin = "https://3.basecampapi.com" + opts.AccountID = "2914079" + opts.ConsumerNamespace = "connector-test" + opts.Ledger = ledger + opts.Queue = queue + opts.Minter = minter + opts.Polls = polls + opts.Transport = transport + intake, err := New(opts) + require.NoError(t, err) + return intake, transport, minter, polls +} + +func ticket() eventfeed.StreamTicket { + return eventfeed.StreamTicket{Ticket: "t", ExpiresIn: 120, URL: "wss://cable.basecamp.com/cable?ticket=t"} +} + +func runInBackground(ctx context.Context, t *testing.T, intake *Intake) chan error { + t.Helper() + done := make(chan error, 1) + go func() { done <- intake.Run(ctx) }() + return done +} + +func awaitReturn(t *testing.T, done chan error, why string) error { + t.Helper() + select { + case err := <-done: + return err + case <-time.After(5 * time.Second): + t.Fatal(why) + return nil + } +} + +func TestATerminalFeedIsNotHeldOpenByTheMembershipWatcher(t *testing.T) { + ledger := newTestLedger(t) + intake, _, minter, _ := newFeedIntake(t, ledger, Options{ + Membership: &flakyMembership{buckets: []int64{48699913}}, + MembershipInterval: time.Hour, + }) + minter.ScriptError(&eventfeed.MintError{Kind: eventfeed.MintUnrecoverable, Err: errors.New("gone")}) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + err := awaitReturn(t, runInBackground(ctx, t, intake), + "a feed that terminated must return its error, not wait out the membership ticker") + assert.Error(t, err) +} + +func TestATerminalFeedIsNotHeldOpenByARepairWalk(t *testing.T) { + ledger := newTestLedger(t) + _, err := ledger.RecordLoss(context.Background(), []int64{17099838509}, time.Now(), time.Hour, eventfeed.Filters{}) + require.NoError(t, err) + + intake, _, minter, _ := newFeedIntake(t, ledger, Options{RepairInterval: time.Hour}) + minter.ScriptError(&eventfeed.MintError{Kind: eventfeed.MintUnrecoverable, Err: errors.New("gone")}) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + err = awaitReturn(t, runInBackground(ctx, t, intake), + "a repair walk is off the delivery path and must not delay the supervisor by its window") + assert.Error(t, err) + + open, err := ledger.OpenLosses(context.Background()) + require.NoError(t, err) + assert.Len(t, open, 1, "the loss stays open for the next start") +} + +func TestAnEventFromAnUnsnapshottedProjectReconnectsPromptly(t *testing.T) { + ledger := newTestLedger(t) + intake, transport, minter, polls := newFeedIntake(t, ledger, Options{ + Membership: &flakyMembership{buckets: []int64{1}}, + MembershipInterval: time.Hour, + }) + minter.ScriptTicket(ticket()) + minter.ScriptTicket(ticket()) + polls.ScriptPage(eventfeed.PollPage{Events: []eventfeed.Event{testEvent(17099838500)}, Position: "p1"}) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := runInBackground(ctx, t, intake) + subscribedConn(t, transport) + + require.Eventually(t, func() bool { return len(transport.Dials()) >= 2 }, 5*time.Second, 10*time.Millisecond, + "the poll lane served project 48699913, which the live subscription does not hold; the reconnect is due now, not at the next membership tick") + + cancel() + awaitReturn(t, done, "Run should return on shutdown") +} + +// After a restart the SDK's own reset cursor starts at zero, so a rejected +// stored position would re-enter at the present. The ledger holds the safe +// re-entry; intake must use it. +func TestARejectedStoredPositionReentersAtThePersistedPollServedID(t *testing.T) { + ledger := newTestLedger(t) + intake, transport, minter, polls := newFeedIntake(t, ledger, Options{}) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + key := intake.CheckpointKey() + require.NoError(t, ledger.Save(ctx, key, "position-the-server-will-refuse")) + require.NoError(t, ledger.NotePollServed(ctx, key, 17099838500)) + + minter.ScriptTicket(ticket()) + minter.ScriptTicket(ticket()) + polls.ScriptError(&eventfeed.PollError{Kind: eventfeed.PollPositionInvalid}) + polls.ScriptPage(eventfeed.PollPage{Position: "p2"}) + polls.ScriptPage(eventfeed.PollPage{Position: "p3"}) + + done := runInBackground(ctx, t, intake) + subscribedConn(t, transport) + + // Whatever the connector does next — re-enter on this connection or make + // a new one — the second poll is the re-entry. A new connection is + // answered as it appears. + answered := 1 + var reentry eventfeed.Cursor + require.Eventually(t, func() bool { + if conns := transport.Conns(); len(conns) > answered { + answerSubscription(conns[len(conns)-1]) + answered = len(conns) + } + calls := polls.Calls() + if len(calls) < 2 { + return false + } + reentry = calls[1].Cursor + return true + }, 5*time.Second, 10*time.Millisecond) + assert.Equal(t, "17099838500", reentry.Since, + "re-entering at the present skips everything between the refused position and now") + + cancel() + awaitReturn(t, done, "Run should return on shutdown") +} + +// A filter change re-enters under the new digest at the last poll-served id, +// which is tracked apart from any one digest's position. +func TestAFilterChangeReentersAtTheLastPollServedID(t *testing.T) { + ledger := newTestLedger(t) + intake, transport, minter, polls := newFeedIntake(t, ledger, Options{ + Filters: eventfeed.Filters{Types: []string{"comment.created"}}, + }) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + old := intake.CheckpointKey() + old.FilterKey = eventfeed.Filters{}.FilterKey() + require.NoError(t, ledger.Save(ctx, old, "old-digest-position")) + require.NoError(t, ledger.NotePollServed(ctx, old, 17099838500)) + + minter.ScriptTicket(ticket()) + polls.ScriptPage(eventfeed.PollPage{Position: "p1"}) + + done := runInBackground(ctx, t, intake) + subscribedConn(t, transport) + + require.Eventually(t, func() bool { return polls.CallCount() > 0 }, 5*time.Second, 10*time.Millisecond) + assert.Equal(t, "17099838500", polls.Calls()[0].Cursor.Since, + "a new filter set with no position of its own must not enter at the present") + + cancel() + awaitReturn(t, done, "Run should return on shutdown") +} + +// answerSubscription greets a connection and confirms its subscription without +// failing the test, so it can run inside a polling condition. +func answerSubscription(conn *feedtest.Conn) { + conn.Serve([]byte(`{"type":"welcome"}`)) + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + if writes := conn.Writes(); len(writes) > 0 { + var command struct { + Identifier string `json:"identifier"` + } + if json.Unmarshal(writes[0], &command) == nil && command.Identifier != "" { + frame, _ := json.Marshal(map[string]string{"type": "confirm_subscription", "identifier": command.Identifier}) + conn.Serve(frame) + return + } + } + time.Sleep(5 * time.Millisecond) + } +} + +// The inbox's 410 shape reaching the repair walk on the account lane is +// refused at the seam, never taken as the epoch's: no epoch gap is recorded +// for it, it is not retried for the window, and the loss stays open. +func TestAnInboxShaped410InTheRepairWalkIsNotTheEpochsPath(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, clock.at, 10*time.Minute, eventfeed.Filters{}) + require.NoError(t, err) + + var hits atomic.Int32 + s := newFeedServer(t, func(w http.ResponseWriter, _ *http.Request, s *feedServer) { + hits.Add(1) + s.json(w, 410, `{"error":"gone","resume":"ORIGIN/2914079/events.json?since=0"}`) + }) + polls, _ := livePolls(t, s) + walker, _ := newTestWalker(t, ledger, polls, clock) + walker.origin = mustOrigin(t, s.URL) + require.NoError(t, walker.reconcile(ctx, loss)) + + gaps, err := ledger.Gaps(ctx) + require.NoError(t, err) + assert.Empty(t, gaps, "no epoch was served, so no epoch gap is recorded") + assert.Equal(t, int32(1), hits.Load(), "a refusal retrying will not fix is not retried for ten minutes") + open, err := ledger.OpenLosses(ctx) + require.NoError(t, err) + assert.Len(t, open, 1) +} + +// The repair walk follows next and resume URLs outside the SDK connector, so +// it holds them to the API origin itself. +func TestTheRepairWalkDoesNotFollowAForeignURL(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, clock.at, 10*time.Minute, eventfeed.Filters{}) + require.NoError(t, err) + + polls := &scriptedPolls{pages: []eventfeed.PollPage{ + {Position: "p1", Next: "https://evil.example.com/2914079/events.json?position=p1"}, + }} + walker, _ := newTestWalker(t, ledger, polls, clock) + require.NoError(t, walker.reconcile(ctx, loss)) + assert.Equal(t, 1, polls.calls, "the foreign next is not followed") + + loss2, err := ledger.RecordLoss(ctx, []int64{17099838600}, clock.at, 10*time.Minute, eventfeed.Filters{}) + require.NoError(t, err) + polls2 := &scriptedPolls{errs: []error{&eventfeed.PollError{Kind: eventfeed.PollGone, EpochAfterID: 17099838550, + ResumeURL: "http://3.basecampapi.com/2914079/events.json?since=17099838550"}}} + walker2, _ := newTestWalker(t, ledger, polls2, clock) + require.NoError(t, walker2.reconcile(ctx, loss2)) + assert.Equal(t, 1, polls2.calls, "a downgraded resume is not followed") +} + +func mustOrigin(t *testing.T, raw string) string { + t.Helper() + origin, err := eventfeed.CanonicalOrigin(raw) + require.NoError(t, err) + return origin +} + +// A 410 in the walk fences off the ids below the epoch. Ids above it are still +// servable from the resume, and giving up on them too would lose them for +// nothing. +func TestAnEpoch410InTheRepairWalkOnlyCondemnsTheIDsBehindIt(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + loss, err := ledger.RecordLoss(ctx, []int64{100, 200}, clock.at, 10*time.Minute, eventfeed.Filters{}) + require.NoError(t, err) + + polls := &scriptedPolls{ + errs: []error{&eventfeed.PollError{ + Kind: eventfeed.PollGone, + EpochAfterID: 150, + ResumeURL: "https://3.basecampapi.com/2914079/events.json?since=150", + }}, + pages: []eventfeed.PollPage{{}, {Events: []eventfeed.Event{testEvent(200)}, Position: "after-epoch"}}, + } + walker, _ := newTestWalker(t, ledger, polls, clock) + require.NoError(t, walker.reconcile(ctx, loss)) + + require.GreaterOrEqual(t, len(polls.cursors), 2) + assert.Equal(t, "https://3.basecampapi.com/2914079/events.json?since=150", polls.cursors[1].PageURL, + "the resume is followed as served") + + unrecovered, err := ledger.UnrecoveredIDs(ctx) + require.NoError(t, err) + assert.Equal(t, []int64{100}, unrecovered) + recovered, err := ledger.MissingIDs(ctx, loss.ID, LossRecovered) + require.NoError(t, err) + assert.Equal(t, []int64{200}, recovered) +} + +// The walk's own cursor can be refused — minted under another filter set, or +// by a rotated secret. The walk re-enters from its explicit id rather than +// retrying the refused cursor for the whole window. +func TestARefusedRepairCursorReentersFromTheExplicitID(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, clock.at, 10*time.Minute, eventfeed.Filters{}) + require.NoError(t, err) + require.NoError(t, ledger.SaveRepairCursor(ctx, loss.ID, "cursor-under-old-filters")) + loss.RepairCursor = "cursor-under-old-filters" + + polls := &scriptedPolls{ + errs: []error{&eventfeed.PollError{Kind: eventfeed.PollFilterChanged}}, + pages: []eventfeed.PollPage{{}, {Events: []eventfeed.Event{testEvent(17099838509)}, Position: "fresh"}}, + } + walker, ingested := newTestWalker(t, ledger, polls, clock) + require.NoError(t, walker.reconcile(ctx, loss)) + + require.GreaterOrEqual(t, len(polls.cursors), 2) + assert.Equal(t, "17099838508", polls.cursors[1].Since) + assert.Equal(t, []int64{17099838509}, *ingested) +} + +// A failure no repeat will fix ends this start's walk but leaves the loss open, +// so the next start — perhaps after the cause is fixed — tries again rather +// than finding the ids already condemned. +func TestAnUnrecoverableRepairPollLeavesTheLossOpen(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, clock.at, 10*time.Minute, eventfeed.Filters{}) + require.NoError(t, err) + + polls := &scriptedPolls{errs: []error{&eventfeed.PollError{Kind: eventfeed.PollRedirectRefused}}} + walker, _ := newTestWalker(t, ledger, polls, clock) + require.NoError(t, walker.reconcile(ctx, loss)) + + assert.Equal(t, 1, polls.calls, "a refused redirect is not worth ten minutes of retries") + open, err := ledger.OpenLosses(ctx) + require.NoError(t, err) + assert.Len(t, open, 1) +} + +// A resume that answers with the same 410 again must not become an endless +// loop that writes a gap per turn. +func TestARepeated410OnTheResumeEndsTheWalk(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + loss, err := ledger.RecordLoss(ctx, []int64{100, 200}, clock.at, 10*time.Minute, eventfeed.Filters{}) + require.NoError(t, err) + + gone := &eventfeed.PollError{Kind: eventfeed.PollGone, EpochAfterID: 150, + ResumeURL: "https://3.basecampapi.com/2914079/events.json?since=150"} + polls := &scriptedPolls{errs: []error{gone, gone, gone, gone, gone, gone, gone, gone, gone, gone, gone, gone, gone, gone, gone, gone, gone, gone, gone, gone, gone, gone, gone, gone, gone, gone, gone, gone, gone, gone}} + walker, _ := newTestWalker(t, ledger, polls, clock) + require.NoError(t, walker.reconcile(ctx, loss)) + + // Eleven passes over the ten-minute window, each at most its entry and + // that entry's resume: bounded by the cadence, not a tight loop. + assert.LessOrEqual(t, polls.calls, 24, "a 410 on the resume is retried on the cadence, not in a tight loop") + gaps, err := ledger.Gaps(ctx) + require.NoError(t, err) + assert.Len(t, gaps, 1, "one 410, one gap") +} diff --git a/internal/connector/round4_test.go b/internal/connector/round4_test.go new file mode 100644 index 000000000..5bca8b681 --- /dev/null +++ b/internal/connector/round4_test.go @@ -0,0 +1,173 @@ +package connector + +import ( + "context" + "strconv" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" +) + +// headPolls models a feed where one event was committed after the stored +// position. The stored position is refused; an entry at the present serves +// nothing; any entry in served history serves the event. +type headPolls struct { + mu sync.Mutex + cursors []eventfeed.Cursor +} + +func (h *headPolls) Poll(_ context.Context, cursor eventfeed.Cursor, _ eventfeed.Filters) (eventfeed.PollPage, error) { + h.mu.Lock() + h.cursors = append(h.cursors, cursor) + h.mu.Unlock() + switch { + case cursor.Position == "checkpoint-from-empty-pages": + return eventfeed.PollPage{}, &eventfeed.PollError{Kind: eventfeed.PollPositionInvalid} + case cursor.Since == "now" || (cursor.Since == "" && cursor.Position == "" && cursor.PageURL == ""): + return eventfeed.PollPage{Position: "head"}, nil + default: + return eventfeed.PollPage{Events: []eventfeed.Event{testEvent(17099838600)}, Position: "after-the-event"}, nil + } +} + +// A position saved from empty pages leaves this filter set's poll-served id at +// zero. If the server then refuses it, the safe re-entry is the beginning of +// served history — not another filter set's id, and not the present. +func TestARefusedPositionWithNoPollServedIDReentersAtTheBeginningNotThePresent(t *testing.T) { + for _, lineage := range []int64{0, 17099838700} { + t.Run("lineage "+strconv.FormatInt(lineage, 10), func(t *testing.T) { + ledger := newTestLedger(t) + polls := &headPolls{} + queue, err := NewQueue(DefaultBacklogWarn, DefaultBacklogPause) + require.NoError(t, err) + intake, transport, minter, _ := newFeedIntake(t, ledger, Options{}) + intake.opts.PollsFor = func() eventfeed.PollSource { return polls } + intake.queue = queue + for range 4 { + minter.ScriptTicket(ticket()) + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + require.NoError(t, ledger.Save(ctx, intake.CheckpointKey(), "checkpoint-from-empty-pages")) + if lineage > 0 { + other := intake.CheckpointKey() + other.FilterKey = "srv2-0000000000000000" + require.NoError(t, ledger.NotePollServed(ctx, other, lineage)) + } + + done := runInBackground(ctx, t, intake) + subscribedConn(t, transport) + + answered := 1 + require.Eventually(t, func() bool { + if conns := transport.Conns(); len(conns) > answered { + answerSubscription(conns[len(conns)-1]) + answered = len(conns) + } + _, ok, err := ledger.Get(ctx, 17099838600) + return err == nil && ok + }, 3*time.Second, 10*time.Millisecond, "the event committed after the refused checkpoint is lost") + + cancel() + awaitReturn(t, done, "Run should return on shutdown") + }) + } +} + +// The ledger's lifecycle is closed; a state outside it, or a reason on a state +// that takes none, would be a durable row no recovery scan can find. +func TestSetStateRefusesAnythingOutsideTheLifecycle(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + _, err := ledger.RecordSeen(ctx, testEvent(1), LanePoll) + require.NoError(t, err) + + assert.Error(t, ledger.SetState(ctx, 1, RecordState("sen"), "")) + assert.Error(t, ledger.SetState(ctx, 1, StateAdmitted, "because")) + assert.Error(t, ledger.SetState(ctx, 1, StateBlocked, ""), "a blocked record says why") + assert.NoError(t, ledger.SetState(ctx, 1, StateBlocked, "read_failed")) + assert.NoError(t, ledger.SetState(ctx, 1, StateAdmitted, "")) +} + +// noncePolls answers every request with the epoch's 410, and each answer's +// resume differs only by a nonce — a server that signs its resume URLs. +type noncePolls struct { + mu sync.Mutex + calls int +} + +func (n *noncePolls) Poll(context.Context, eventfeed.Cursor, eventfeed.Filters) (eventfeed.PollPage, error) { + n.mu.Lock() + defer n.mu.Unlock() + n.calls++ + if n.calls > 500 { + return eventfeed.PollPage{}, &eventfeed.PollError{Kind: eventfeed.PollUnrecoverable} + } + return eventfeed.PollPage{}, &eventfeed.PollError{Kind: eventfeed.PollGone, EpochAfterID: 150, + ResumeURL: "https://3.basecampapi.com/2914079/events.json?since=150&nonce=" + strconv.Itoa(n.calls)} +} + +// C3: a pass is bounded whatever URLs the server chooses. +func TestARepairPassIsBoundedWhenEveryResumeURLDiffers(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + loss, err := ledger.RecordLoss(ctx, []int64{100, 200}, clock.at, time.Minute, eventfeed.Filters{}) + require.NoError(t, err) + + polls := &noncePolls{} + walker, _ := newTestWalker(t, ledger, polls, clock) + require.NoError(t, walker.reconcile(ctx, loss)) + assert.LessOrEqual(t, polls.calls, 10, "two passes, each a handful of polls, not a hot loop") +} + +// A ledger read that failed because the connection was canceled — by a +// reconnect — is not a reason to end the run. +func TestACanceledReadDuringAReconnectIsNotAnAbort(t *testing.T) { + intake, _, _ := newTestIntake(t, nil, nil) + intake.positions = canceledPositions{} + ctx, cancel := context.WithCancel(context.Background()) + cancel() + intake.onPositionRejected(ctx) + assert.NoError(t, intake.abortErr) +} + +type canceledPositions struct{} + +func (canceledPositions) LastPollServedID(ctx context.Context, _ eventfeed.CheckpointKey) (int64, error) { + return 0, ctx.Err() +} + +func (canceledPositions) LineagePollServedID(ctx context.Context, _ eventfeed.CheckpointKey) (int64, error) { + return 0, ctx.Err() +} + +// Queue callbacks may observe the queue. None of them runs while the queue +// holds a lock its own operations need. +func TestQueueCallbacksMayTouchTheQueue(t *testing.T) { + queue, err := NewQueue(1, 4) + require.NoError(t, err) + var depths []int + // No timeout: a callback that takes from the queue must actually get the + // id the offer is delivering, not time out and call that success. + queue.OnWarn = func(int) { + id, err := queue.Take(context.Background()) + assert.NoError(t, err) + assert.Equal(t, int64(1), id) + depths = append(depths, queue.Depth()) + } + + done := make(chan error, 1) + go func() { done <- queue.Offer(context.Background(), 1) }() + select { + case err := <-done: + require.NoError(t, err) + case <-time.After(2 * time.Second): + t.Fatal("a callback that takes from the queue deadlocked the offer") + } +} diff --git a/internal/connector/round5_test.go b/internal/connector/round5_test.go new file mode 100644 index 000000000..9beeaa3c5 --- /dev/null +++ b/internal/connector/round5_test.go @@ -0,0 +1,113 @@ +package connector + +import ( + "context" + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" +) + +// A3: the last poll-served id counts what the poll lane SERVED, not what it +// delivered. In steady state the poll copy of nearly every event is +// suppressed, because the live lane delivered it thirty seconds earlier; +// counting deliveries leaves the id at zero and every later re-entry falls +// back to a full replay — or, on a filter change, to the present. +func TestThePollServedIDCountsServedEventsTheLiveLaneAlreadyDelivered(t *testing.T) { + ledger := newTestLedger(t) + intake, transport, minter, polls := newFeedIntake(t, ledger, Options{RepairInterval: 50 * time.Millisecond}) + minter.ScriptTicket(ticket()) + polls.ScriptPage(eventfeed.PollPage{Position: "p1"}) + polls.ScriptPage(eventfeed.PollPage{Events: []eventfeed.Event{testEvent(17099838600)}, Position: "p2"}) + for range 20 { + polls.ScriptPage(eventfeed.PollPage{Position: "p2"}) + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := runInBackground(ctx, t, intake) + conn, identifier := subscribedConn(t, transport) + + require.Eventually(t, func() bool { return polls.CallCount() >= 1 }, 5*time.Second, time.Millisecond) + conn.Serve(pushFrame(t, identifier, testEvent(17099838600))) + require.Eventually(t, func() bool { + _, ok, err := ledger.Get(ctx, 17099838600) + return err == nil && ok + }, 5*time.Second, time.Millisecond) + + require.Eventually(t, func() bool { + served, err := ledger.LastPollServedID(ctx, intake.CheckpointKey()) + return err == nil && served == 17099838600 + }, 5*time.Second, 5*time.Millisecond, "the repair poll served the event; its delivery being suppressed does not unserve it") + + cancel() + awaitReturn(t, done, "Run should return on shutdown") +} + +// A3: a filter change enters under the new digest at the old lineage's +// poll-served id — and, when the lineage has a position but no id, at the +// beginning of served history. Never at the present. +func TestAFilterChangeFromAPositionWithNoServedIDReplaysInsteadOfEnteringAtThePresent(t *testing.T) { + ledger := newTestLedger(t) + intake, transport, minter, polls := newFeedIntake(t, ledger, Options{ + Filters: eventfeed.Filters{Types: []string{"comment.created"}}, + }) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + old := intake.CheckpointKey() + old.FilterKey = eventfeed.Filters{}.FilterKey() + require.NoError(t, ledger.Save(ctx, old, "old-digest-position-from-empty-pages")) + + minter.ScriptTicket(ticket()) + polls.ScriptPage(eventfeed.PollPage{Position: "p1"}) + + done := runInBackground(ctx, t, intake) + subscribedConn(t, transport) + + require.Eventually(t, func() bool { return polls.CallCount() > 0 }, 5*time.Second, 10*time.Millisecond) + first := polls.Calls()[0].Cursor + assert.Equal(t, "0", first.Since, "the old filter set had read up to a position; entering at the present skips what followed it") + + cancel() + awaitReturn(t, done, "Run should return on shutdown") +} + +// A gap met by a deliberate recovery replay is labeled as one, so status does +// not present a replay's expected 410 as a loss. +func TestAGapMetByARecoveryReplayIsLabeledAsOne(t *testing.T) { + intake, ledger, _ := newTestIntake(t, nil, nil) + intake.replaying = true + + assert.Equal(t, eventfeed.Accept, intake.handleSignal(eventfeed.FeedGap{ + EpochAfterID: 17099838487, + ResumeURL: "https://3.basecampapi.com/2914079/events.json?since=17099838487", + })) + gaps, err := ledger.Gaps(context.Background()) + require.NoError(t, err) + require.Len(t, gaps, 1) + assert.Contains(t, gaps[0].Note, "replay") +} + +// The replay label covers the replay's own entry, not the life of the +// connection: a later, genuine epoch move must not be recorded as expected. +func TestTheReplayLabelCoversOnlyTheReplaysOwnGap(t *testing.T) { + intake, ledger, _ := newTestIntake(t, nil, nil) + intake.replaying = true + + for _, epoch := range []int64{100, 17099838000} { + assert.Equal(t, eventfeed.Accept, intake.handleSignal(eventfeed.FeedGap{ + EpochAfterID: epoch, + ResumeURL: "https://3.basecampapi.com/2914079/events.json?since=" + strconv.FormatInt(epoch, 10), + })) + } + gaps, err := ledger.Gaps(context.Background()) + require.NoError(t, err) + require.Len(t, gaps, 2) + assert.Contains(t, gaps[0].Note, "replay") + assert.NotContains(t, gaps[1].Note, "not a loss", "a later epoch move is a real gap") +} diff --git a/internal/connector/round6_test.go b/internal/connector/round6_test.go new file mode 100644 index 000000000..849797d3e --- /dev/null +++ b/internal/connector/round6_test.go @@ -0,0 +1,95 @@ +package connector + +import ( + "bytes" + "context" + "encoding/json" + "log/slog" + "strings" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" +) + +// F2: an operator sees warning edges in the order the state took them. +// Delivered out of order, a drained queue can end on a warning. +func TestQueueWarningEdgesArriveInTheOrderTheyHappened(t *testing.T) { + for range 30 { + queue, err := NewQueue(1, 64) + require.NoError(t, err) + var mu sync.Mutex + var seen []string + queue.OnWarn = func(int) { mu.Lock(); seen = append(seen, "warn"); mu.Unlock() } + queue.OnRecover = func(int) { mu.Lock(); seen = append(seen, "recover"); mu.Unlock() } + + ctx := context.Background() + var wg sync.WaitGroup + for g := range 8 { + wg.Add(1) + go func() { + defer wg.Done() + for i := range 300 { + require.NoError(t, queue.Offer(ctx, int64(g*1000+i))) + _, err := queue.Take(ctx) + require.NoError(t, err) + } + }() + } + wg.Wait() + + mu.Lock() + for i, edge := range seen { + want := "warn" + if i%2 == 1 { + want = "recover" + } + require.Equal(t, want, edge, "edge %d of %v", i, len(seen)) + } + if len(seen) > 0 { + require.Equal(t, "recover", seen[len(seen)-1], "a drained queue's last word is a recovery") + } + mu.Unlock() + } +} + +// C2: a loss that ended with ids behind the epoch is not reported as fully +// reconciled. +func TestAPartlyUnrecoveredLossIsNotReportedAsFullyReconciled(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + loss, err := ledger.RecordLoss(ctx, []int64{100, 200}, clock.at, 10*time.Minute, eventfeed.Filters{}) + require.NoError(t, err) + + polls := &scriptedPolls{ + errs: []error{&eventfeed.PollError{Kind: eventfeed.PollGone, EpochAfterID: 150, + ResumeURL: "https://3.basecampapi.com/2914079/events.json?since=150"}}, + pages: []eventfeed.PollPage{{}, {Events: []eventfeed.Event{testEvent(200)}, Position: "p"}}, + } + var logs bytes.Buffer + walker, _ := newTestWalker(t, ledger, polls, clock) + walker.log = slog.New(slog.NewTextHandler(&logs, nil)) + require.NoError(t, walker.reconcile(ctx, loss)) + + assert.NotContains(t, logs.String(), "fully reconciled", "id 100 is behind the epoch and was never recovered") +} + +// The pointer line keeps the event's timestamp at the precision the feed gave. +func TestPointerLineKeepsTheTimestampsPrecision(t *testing.T) { + var pointers bytes.Buffer + intake, _, _ := newTestIntake(t, nil, &pointers) + event := testEvent(1) + event.CreatedAt = time.Date(2026, 9, 16, 10, 0, 0, 120_000_000, time.UTC) + require.NoError(t, intake.ingest(context.Background(), event, LanePoll)) + + var pointer Pointer + require.NoError(t, json.Unmarshal([]byte(strings.TrimSpace(pointers.String())), &pointer)) + parsed, err := time.Parse(time.RFC3339Nano, pointer.CreatedAt) + require.NoError(t, err) + assert.True(t, parsed.Equal(event.CreatedAt), "got %s", pointer.CreatedAt) +} diff --git a/internal/connector/round7_test.go b/internal/connector/round7_test.go new file mode 100644 index 000000000..b55e73c6d --- /dev/null +++ b/internal/connector/round7_test.go @@ -0,0 +1,66 @@ +package connector + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Retention compares timestamps. Stored as variable-width text, a whole-second +// stamp ("…00Z") sorts after a later fractional one ("…00.5Z") in the same +// second, and a record past its window is kept. +func TestDropContentDropsARecordExpiredWithinTheSameSecond(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + whole := time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC) + ledger.now = func() time.Time { return whole } + + _, err := ledger.RecordSeen(ctx, testEvent(42), LanePoll) + require.NoError(t, err) + require.NoError(t, ledger.SetState(ctx, 42, StateDiscarded, "untrusted_author")) + + cutoff := whole.Add(500 * time.Millisecond) + dropped, err := ledger.DropContent(ctx, cutoff, cutoff) + require.NoError(t, err) + assert.Equal(t, 1, dropped, "updated at 12:00:00.000, cutoff 12:00:00.500: expired") +} + +// The other direction: a record updated half a second AFTER a whole-second +// cutoff is inside its window and must be kept. As variable-width text, +// "…00.5Z" < "…00Z", so it would be purged. +func TestDropContentKeepsARecordUpdatedJustAfterAWholeSecondCutoff(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + cutoff := time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC) + ledger.now = func() time.Time { return cutoff.Add(500 * time.Millisecond) } + + _, err := ledger.RecordSeen(ctx, testEvent(43), LanePoll) + require.NoError(t, err) + // The lifecycle has no shortcut to completed, so the record walks there. + require.NoError(t, ledger.SetState(ctx, 43, StateAdmitted, "")) + require.NoError(t, ledger.SetState(ctx, 43, StateDispatched, "")) + require.NoError(t, ledger.SetState(ctx, 43, StateCompleted, "")) + + dropped, err := ledger.DropContent(ctx, cutoff, cutoff) + require.NoError(t, err) + assert.Zero(t, dropped, "updated at 12:00:00.500, cutoff 12:00:00.000: still inside the window") +} + +// Every stored timestamp has one width, which is the property both directions +// rest on. +func TestLedgerTimestampsHaveOneWidth(t *testing.T) { + for _, at := range []time.Time{ + time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC), + time.Date(2026, 9, 16, 12, 0, 0, 500_000_000, time.FixedZone("CEST", 2*3600)), + time.Date(2026, 9, 16, 12, 0, 0, 123_456_789, time.UTC), + } { + s := stamp(at) + assert.Len(t, s, len("2006-01-02T15:04:05.000000000Z"), s) + parsed, err := parseStamp(s) + require.NoError(t, err) + assert.True(t, parsed.Equal(at)) + } +} diff --git a/internal/connector/round8_test.go b/internal/connector/round8_test.go new file mode 100644 index 000000000..39dd62684 --- /dev/null +++ b/internal/connector/round8_test.go @@ -0,0 +1,97 @@ +package connector + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" +) + +// A3: a missing position does not mean a new filter. The package confirms a +// page before it saves the position, so a failed save leaves this filter's +// poll-served id recorded with no position. A restart re-enters at that id, +// not at another filter set's larger one. +func TestARestartWithoutAPositionPrefersThisFilterSetsOwnServedID(t *testing.T) { + ledger := newTestLedger(t) + intake, transport, minter, polls := newFeedIntake(t, ledger, Options{}) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + require.NoError(t, ledger.NotePollServed(ctx, intake.CheckpointKey(), 1000)) + other := intake.CheckpointKey() + other.FilterKey = "srv2-0000000000000000" + require.NoError(t, ledger.NotePollServed(ctx, other, 2000)) + + minter.ScriptTicket(ticket()) + polls.ScriptPage(eventfeed.PollPage{Position: "p1"}) + + done := runInBackground(ctx, t, intake) + subscribedConn(t, transport) + require.Eventually(t, func() bool { return polls.CallCount() > 0 }, 5*time.Second, 10*time.Millisecond) + assert.Equal(t, "1000", polls.Calls()[0].Cursor.Since, + "another filter set's id is past events this one never served") + + cancel() + awaitReturn(t, done, "Run should return on shutdown") +} + +// E2: a safe re-entry that is itself refused ends the run. Reconnecting again +// would mint, dial and poll in a tight loop against the API. +func TestARefusedReentryEndsTheRunInsteadOfLooping(t *testing.T) { + ledger := newTestLedger(t) + intake, transport, minter, polls := newFeedIntake(t, ledger, Options{}) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + require.NoError(t, ledger.Save(ctx, intake.CheckpointKey(), "refused")) + require.NoError(t, ledger.NotePollServed(ctx, intake.CheckpointKey(), 1000)) + + for range 50 { + minter.ScriptTicket(ticket()) + polls.ScriptError(&eventfeed.PollError{Kind: eventfeed.PollPositionInvalid}) + } + + done := runInBackground(ctx, t, intake) + answered := 0 + var err error + require.Eventually(t, func() bool { + if conns := transport.Conns(); len(conns) > answered { + answerSubscription(conns[len(conns)-1]) + answered = len(conns) + } + select { + case err = <-done: + return true + default: + return false + } + }, 5*time.Second, 10*time.Millisecond, "a refused re-entry must end the run") + assert.Error(t, err) + assert.LessOrEqual(t, minter.Calls(), 3, "one refusal, one re-entry, one refusal of that: then stop") +} + +// C2: a loss whose repair can never finish — every pass ends in a failure no +// retry fixes — is still closed once its window has passed, so status shows its +// ids as unrecovered rather than open forever. +func TestALossPastItsWindowClosesEvenWhenItsWalkCannotFinish(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, clock.at.Add(-24*time.Hour), 10*time.Minute, eventfeed.Filters{}) + require.NoError(t, err) + + polls := &scriptedPolls{errs: []error{&eventfeed.PollError{Kind: eventfeed.PollFilterInvalid, Err: errors.New("x")}}} + walker, _ := newTestWalker(t, ledger, polls, clock) + require.NoError(t, walker.reconcile(ctx, loss)) + + open, err := ledger.OpenLosses(ctx) + require.NoError(t, err) + assert.Empty(t, open) + unrecovered, err := ledger.UnrecoveredIDs(ctx) + require.NoError(t, err) + assert.Equal(t, []int64{17099838509}, unrecovered) +} diff --git a/internal/connector/round9_test.go b/internal/connector/round9_test.go new file mode 100644 index 000000000..ebcb530d4 --- /dev/null +++ b/internal/connector/round9_test.go @@ -0,0 +1,255 @@ +package connector + +import ( + "bytes" + "context" + "errors" + "log/slog" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" +) + +// The filter set is the checkpoint's identity. A caller that keeps its slice +// and mutates it would change the filters intake subscribes, records and +// walks with, while the checkpoint key stays frozen on what it was built from. +func TestTheFilterSetIsCopiedAtConstruction(t *testing.T) { + ledger := newTestLedger(t) + queue, err := NewQueue(1, 2) + require.NoError(t, err) + types := []string{"comment.created"} + buckets := []int64{48699913} + intake, err := New(Options{ + Origin: "https://3.basecampapi.com", + AccountID: "2914079", + ConsumerNamespace: "connector-test", + Filters: eventfeed.Filters{Types: types, Buckets: buckets}, + Ledger: ledger, + Queue: queue, + Minter: stubMinter{}, + Polls: &scriptedPolls{}, + }) + require.NoError(t, err) + + types[0] = "card.created" + buckets[0] = 999 + + assert.Equal(t, []string{"comment.created"}, intake.opts.Filters.Types) + assert.Equal(t, []int64{48699913}, intake.opts.Filters.Buckets) + assert.Equal(t, eventfeed.Filters{Types: []string{"comment.created"}, Buckets: []int64{48699913}}.FilterKey(), + intake.CheckpointKey().FilterKey) +} + +// A peer's disconnect reason is text the other end chose. It reaches an +// operator's terminal through the log. +func TestALoggedDisconnectReasonCarriesNoTerminalControls(t *testing.T) { + var logs bytes.Buffer + intake, _, _ := newTestIntake(t, nil, nil) + intake.log = slog.New(slog.NewJSONHandler(&logs, nil)) + + observer := intake.observer(context.Background()) + require.NotNil(t, observer.Disconnected) + observer.Disconnected("stale"+csi+"31m"+esc+"[2J", nil) + + assert.NotContains(t, logs.String(), csi) + assert.NotContains(t, logs.String(), esc) + assert.Contains(t, logs.String(), "stale") +} + +// A re-entry that saved a checkpoint has made progress, even if the page it +// saved carried no event. A later refusal of that saved position is an +// ordinary refusal, not the re-entry being refused. +func TestAReentryThatCheckpointedAnEmptyPageIsNotStillPending(t *testing.T) { + ledger := newTestLedger(t) + // A short repair cadence so the position saved from the empty page is + // re-polled, and refused, while this connection is streaming. + intake, transport, minter, polls := newFeedIntake(t, ledger, Options{RepairInterval: 50 * time.Millisecond}) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + require.NoError(t, ledger.Save(ctx, intake.CheckpointKey(), "refused")) + require.NoError(t, ledger.NotePollServed(ctx, intake.CheckpointKey(), 1000)) + + for range 6 { + minter.ScriptTicket(ticket()) + } + // The stored position is refused; the re-entry's first page is empty but + // checkpoints; that saved position is then refused in turn. + polls.ScriptError(&eventfeed.PollError{Kind: eventfeed.PollPositionInvalid}) + polls.ScriptPage(eventfeed.PollPage{Position: "saved-from-an-empty-page"}) + polls.ScriptError(&eventfeed.PollError{Kind: eventfeed.PollPositionInvalid}) + polls.ScriptPage(eventfeed.PollPage{Events: []eventfeed.Event{testEvent(17099838600)}, Position: "p3"}) + for range 4 { + polls.ScriptPage(eventfeed.PollPage{Position: "p4"}) + } + + done := runInBackground(ctx, t, intake) + answered := 0 + require.Eventually(t, func() bool { + if conns := transport.Conns(); len(conns) > answered { + answerSubscription(conns[len(conns)-1]) + answered = len(conns) + } + _, ok, err := ledger.Get(ctx, 17099838600) + return err == nil && ok + }, 5*time.Second, 10*time.Millisecond, "the run aborted instead of re-entering after a checkpointed empty page") + + cancel() + awaitReturn(t, done, "Run should return on shutdown") +} + +// countingPolls reports the highest number of polls in flight at once. +type countingPolls struct { + inFlight atomic.Int32 + peak atomic.Int32 + calls atomic.Int32 +} + +func (c *countingPolls) Poll(context.Context, eventfeed.Cursor, eventfeed.Filters) (eventfeed.PollPage, error) { + n := c.inFlight.Add(1) + for { + peak := c.peak.Load() + if n <= peak || c.peak.CompareAndSwap(peak, n) { + break + } + } + defer c.inFlight.Add(-1) + c.calls.Add(1) + time.Sleep(5 * time.Millisecond) + return eventfeed.PollPage{Position: "p"}, nil +} + +// C3: many open losses are repaired within a bound. One goroutine and one poll +// source per loss would turn an overloaded feed into an API storm. +func TestRepairsRunWithinABound(t *testing.T) { + polls := &countingPolls{} + intake, ledger, _ := newTestIntake(t, polls, nil) + intake.opts.RepairInterval = time.Hour + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + for i := range 25 { + // A window already closed, so each loss is one final pass and done. + _, err := ledger.RecordLoss(ctx, []int64{int64(1000 + i)}, intake.now().Add(-time.Hour), time.Minute, eventfeed.Filters{}) + require.NoError(t, err) + } + require.NoError(t, intake.resumeReconciliation(ctx)) + + require.Eventually(t, func() bool { return polls.calls.Load() >= 10 }, 5*time.Second, 5*time.Millisecond) + assert.LessOrEqual(t, int(polls.peak.Load()), maxConcurrentRepairs, "repairs are bounded") + + cancel() + intake.repairs.Wait() +} + +// The ledger is a file. SQLite's in-memory URI would accept every write and +// lose it on close, which is the one thing the ledger exists not to do. +func TestAnInMemoryLedgerIsRefused(t *testing.T) { + for _, path := range []string{":memory:", "file::memory:"} { + _, err := OpenLedger(path) + require.Error(t, err, path) + assert.Contains(t, strings.ToLower(err.Error()), "memory") + } + // A real path containing the word is still a file. + ledger, err := OpenLedger(filepath.Join(t.TempDir(), "state", "memory.db")) + require.NoError(t, err) + require.NoError(t, ledger.Close()) +} + +// Each Run gets its own repair pool. A pool left over from a finished run has +// no workers, so a second Run would queue losses nobody walks — silently. +func TestASecondRunRepairsToo(t *testing.T) { + polls := &countingPolls{} + intake, ledger, _ := newTestIntake(t, polls, nil) + intake.opts.RepairInterval = time.Hour + intake.opts.Minter = stubMinter{} + + record := func() { + _, err := ledger.RecordLoss(context.Background(), []int64{17099838509}, intake.now().Add(-time.Hour), time.Minute, eventfeed.Filters{}) + require.NoError(t, err) + } + + first, cancelFirst := context.WithCancel(context.Background()) + record() + require.NoError(t, intake.resumeReconciliation(first)) + require.Eventually(t, func() bool { return polls.calls.Load() >= 1 }, 5*time.Second, 5*time.Millisecond) + cancelFirst() + intake.repairs.Wait() + intake.releaseRepairWorkers() + after := polls.calls.Load() + + second, cancelSecond := context.WithCancel(context.Background()) + defer cancelSecond() + record() + require.NoError(t, intake.resumeReconciliation(second)) + require.Eventually(t, func() bool { return polls.calls.Load() > after }, 5*time.Second, 5*time.Millisecond, + "the second run's repairs are queued to a pool with no workers") +} + +// Run is reusable, so its per-run state starts clean. A checkpoint saved by an +// earlier run would otherwise clear this run's --since before it has saved +// anything of its own, and an earlier abort would end it before it began. +func TestASecondRunStartsWithCleanRunState(t *testing.T) { + ledger := newTestLedger(t) + intake, transport, minter, polls := newFeedIntake(t, ledger, Options{SinceEventID: 17099838000}) + intake.checkpointed = true + intake.abortErr = errors.New("an earlier run's fatal") + intake.hasReentry = true + intake.replaying = true + intake.enteredByReentry = true + + minter.ScriptTicket(ticket()) + polls.ScriptPage(eventfeed.PollPage{Position: "p1"}) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := runInBackground(ctx, t, intake) + subscribedConn(t, transport) + + require.Eventually(t, func() bool { return polls.CallCount() > 0 }, 5*time.Second, 10*time.Millisecond, + "an earlier run's abort must not end this one") + assert.Equal(t, "17099838000", polls.Calls()[0].Cursor.Since) + + // checkpointed is legitimately true again by now, this run having saved + // its own page; what must not come back is the rest. + intake.mu.Lock() + clean := intake.abortErr == nil && !intake.hasReentry && !intake.replaying + intake.mu.Unlock() + assert.True(t, clean, "per-run state starts clean") + + cancel() + awaitReturn(t, done, "Run should return on shutdown") +} + +// A loss still in the queue when a run ends is walked by the next one: the +// mark that says "a worker has this" belongs to the pool, and goes with it. +func TestALossQueuedAtShutdownIsWalkedByTheNextRun(t *testing.T) { + polls := &countingPolls{} + intake, ledger, _ := newTestIntake(t, polls, nil) + intake.opts.RepairInterval = time.Hour + intake.repairSweep = time.Hour + + first, cancelFirst := context.WithCancel(context.Background()) + intake.startRepairWorkers(first) + loss, err := ledger.RecordLoss(first, []int64{17099838509}, intake.now().Add(-time.Hour), time.Minute, eventfeed.Filters{}) + require.NoError(t, err) + cancelFirst() + intake.repairs.Wait() + intake.startRepair(loss) // queued, with no worker left to take it + intake.releaseRepairWorkers() + + second, cancelSecond := context.WithCancel(context.Background()) + defer cancelSecond() + intake.startRepairWorkers(second) + intake.startRepair(loss) + require.Eventually(t, func() bool { return polls.calls.Load() > 0 }, 5*time.Second, 5*time.Millisecond, + "the next run walks it") + cancelSecond() + intake.repairs.Wait() +} diff --git a/internal/connector/seam_contract_test.go b/internal/connector/seam_contract_test.go new file mode 100644 index 000000000..a41d9ebf0 --- /dev/null +++ b/internal/connector/seam_contract_test.go @@ -0,0 +1,250 @@ +package connector + +import ( + "bytes" + "context" + "errors" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp" + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" +) + +// These pin, at the real seam intake is built on (eventfeed.NewLive over the +// generated operations), the contract its recovery paths depend on. The +// handling lives in basecamp-sdk now; if the SDK ever changes it, intake's +// assumptions break here rather than silently in production. + +type feedServer struct { + *httptest.Server + foreignURL string + foreignHits atomic.Int32 +} + +func newFeedServer(t *testing.T, handler func(w http.ResponseWriter, r *http.Request, s *feedServer)) *feedServer { + t.Helper() + s := &feedServer{} + foreign := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + s.foreignHits.Add(1) + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(foreign.Close) + s.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + handler(w, r, s) + })) + t.Cleanup(s.Close) + s.foreignURL = foreign.URL + return s +} + +func (s *feedServer) json(w http.ResponseWriter, status int, body string) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _, _ = w.Write([]byte(strings.ReplaceAll(body, "ORIGIN", s.URL))) +} + +func livePolls(t *testing.T, s *feedServer) (eventfeed.PollSource, eventfeed.TicketMinter) { + t.Helper() + live, err := eventfeed.NewLive(&basecamp.Config{BaseURL: s.URL}, &basecamp.StaticTokenProvider{Token: "token"}, + "2914079", eventfeed.AccountLane, basecamp.WithMaxRetries(0)) + require.NoError(t, err) + opts := LiveOptions(live) + return opts.PollsFor(), opts.Minter +} + +func pollOnce(t *testing.T, status int, body string) *eventfeed.PollError { + t.Helper() + s := newFeedServer(t, func(w http.ResponseWriter, _ *http.Request, s *feedServer) { s.json(w, status, body) }) + polls, _ := livePolls(t, s) + _, err := polls.Poll(context.Background(), eventfeed.Cursor{Position: "p"}, eventfeed.Filters{}) + var pollErr *eventfeed.PollError + require.ErrorAs(t, err, &pollErr) + return pollErr +} + +// D1: the feed's 410 names its epoch and becomes the gap signal. +func TestSeamTheFeedsEpoch410IsTheGapSignal(t *testing.T) { + pollErr := pollOnce(t, 410, `{"error":"gone","epoch_after_id":17099838487,"resume":"ORIGIN/2914079/events.json?since=17099838487"}`) + assert.Equal(t, eventfeed.PollGone, pollErr.Kind) + assert.Equal(t, int64(17099838487), pollErr.EpochAfterID) + assert.True(t, strings.HasSuffix(pollErr.ResumeURL, "since=17099838487")) +} + +// D1: the inbox's 410 shape — no epoch, a since=0 resume — arriving on the +// account lane never becomes the gap signal, and no epoch is invented for it. +func TestSeamAnInboxShaped410OnTheAccountLaneIsNeverTheGapSignal(t *testing.T) { + pollErr := pollOnce(t, 410, `{"error":"gone","resume":"ORIGIN/2914079/events.json?since=0"}`) + assert.NotEqual(t, eventfeed.PollGone, pollErr.Kind, "a retention loss must not take the epoch's recovery path") + assert.Zero(t, pollErr.EpochAfterID) +} + +// D1: a 410 whose resume does not re-enter at the fence it declares is refused +// rather than followed into a skip. +func TestSeamA410WhoseResumeSkipsTheFenceIsRefused(t *testing.T) { + pollErr := pollOnce(t, 410, `{"error":"gone","epoch_after_id":7,"resume":"ORIGIN/2914079/events.json?since=now"}`) + assert.NotEqual(t, eventfeed.PollGone, pollErr.Kind) +} + +// D3: the 400's reason keys recover-versus-stop. +func TestSeamThe400sReasonDecidesRecoverOrStop(t *testing.T) { + assert.Equal(t, eventfeed.PollPositionInvalid, + pollOnce(t, 400, `{"error":"x","reason":"invalid_position"}`).Kind) + assert.Equal(t, eventfeed.PollFilterInvalid, + pollOnce(t, 400, `{"error":"x","reason":"invalid_filter"}`).Kind) + unknown := pollOnce(t, 400, `{"error":"Unrecognized position","reason":"invalid_something"}`).Kind + assert.NotEqual(t, eventfeed.PollPositionInvalid, unknown, "an unnamed reason is surfaced, never guessed") + assert.NotEqual(t, eventfeed.PollFilterInvalid, unknown) +} + +// D2: a followed URL carries exactly one cursor; with neither it would enter at +// the present. +func TestSeamACursorlessContinuationIsRefusedBeforeAnyRequest(t *testing.T) { + var hits atomic.Int32 + s := newFeedServer(t, func(w http.ResponseWriter, _ *http.Request, s *feedServer) { + hits.Add(1) + s.json(w, 200, `{"events":[],"position":"p"}`) + }) + polls, _ := livePolls(t, s) + _, err := polls.Poll(context.Background(), eventfeed.Cursor{PageURL: s.URL + "/2914079/events.json"}, eventfeed.Filters{}) + var pollErr *eventfeed.PollError + require.ErrorAs(t, err, &pollErr) + assert.Zero(t, hits.Load()) +} + +// D4 / H1: a 3xx is never followed, same origin or not, and the target is not +// rendered. +func TestSeamRedirectsAreRefusedWithZeroEgress(t *testing.T) { + s := newFeedServer(t, func(w http.ResponseWriter, r *http.Request, s *feedServer) { + http.Redirect(w, r, s.foreignURL+"/steal?leak=SECRET-TARGET", http.StatusFound) + }) + polls, minter := livePolls(t, s) + + _, err := polls.Poll(context.Background(), eventfeed.Cursor{Position: "p"}, eventfeed.Filters{}) + var pollErr *eventfeed.PollError + require.ErrorAs(t, err, &pollErr) + assert.Equal(t, eventfeed.PollRedirectRefused, pollErr.Kind) + assert.NotContains(t, err.Error(), "SECRET-TARGET") + + _, err = minter.MintStreamTicket(context.Background()) + require.Error(t, err) + assert.NotContains(t, err.Error(), "SECRET-TARGET") + + assert.Zero(t, s.foreignHits.Load(), "no request may reach the redirect target") +} + +// E4: the caller's own cancellation passes through unclassified. +func TestSeamCallerCancellationIsNotATransportFailure(t *testing.T) { + s := newFeedServer(t, func(w http.ResponseWriter, _ *http.Request, s *feedServer) { + s.json(w, 200, `{"events":[],"position":"p"}`) + }) + polls, _ := livePolls(t, s) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := polls.Poll(ctx, eventfeed.Cursor{Position: "p"}, eventfeed.Filters{}) + var pollErr *eventfeed.PollError + assert.False(t, errors.As(err, &pollErr)) +} + +// Reasons filter the inbox — why an event reached you — and the account feed +// does not carry them. The refusal is here rather than at the feed, because +// Run starts the repair workers before the feed exists: a loss walking under +// a filter set the lane cannot honor reads WIDER than the caller asked for. +func TestIntakeRefusesAnInboxOnlyFilter(t *testing.T) { + queue, err := NewQueue(DefaultBacklogWarn, DefaultBacklogPause) + require.NoError(t, err) + + _, err = New(Options{ + Origin: "https://3.basecampapi.com", + AccountID: "2914079", + ConsumerNamespace: "connector-test", + Ledger: newTestLedger(t), + Queue: queue, + Minter: stubMinter{}, + Polls: &scriptedPolls{}, + Filters: eventfeed.Filters{Reasons: []string{"mention"}}, + }) + + require.Error(t, err) + assert.Contains(t, err.Error(), "reasons filter the inbox") +} + +// The queue reports a callback that panicked, and intake owns the logger that +// says where. +func TestIntakeGivesTheQueueItsLogger(t *testing.T) { + var logs bytes.Buffer + log := slog.New(slog.NewTextHandler(&logs, nil)) + queue, err := NewQueue(1, 4) + require.NoError(t, err) + _, err = New(Options{ + Origin: "https://3.basecampapi.com", + AccountID: "2914079", + ConsumerNamespace: "connector-test", + Ledger: newTestLedger(t), + Queue: queue, + Minter: stubMinter{}, + Polls: &scriptedPolls{}, + Logger: log, + }) + require.NoError(t, err) + + queue.OnWarn = func(int) { panic("a callback panics") } + require.NoError(t, queue.Offer(context.Background(), 1)) + + assert.Contains(t, logs.String(), "a backlog callback panicked") +} + +// A queue can already be in use when intake adopts it — admission takes from +// the same queue — so adopting its logger must not race the callback path that +// reads it. +func TestAdoptingAQueueInUseDoesNotRace(t *testing.T) { + queue, err := NewQueue(1, 1000) + require.NoError(t, err) + queue.OnWarn = func(int) { panic("a callback panics") } + queue.OnRecover = func(int) { panic("a callback panics") } + ledger := newTestLedger(t) + + ctx := context.Background() + stop := make(chan struct{}) + var wg sync.WaitGroup + // The queue is in use, and every crossing reads its logger, for the + // whole time intake is adopting it. + wg.Go(func() { + for { + select { + case <-stop: + return + default: + } + if err := queue.Offer(ctx, 1); err != nil { + return + } + if _, err := queue.Take(ctx); err != nil { + return + } + } + }) + for range 20 { + _, err := New(Options{ + Origin: "https://3.basecampapi.com", + AccountID: "2914079", + ConsumerNamespace: "connector-test", + Ledger: ledger, + Queue: queue, + Minter: stubMinter{}, + Polls: &scriptedPolls{}, + Logger: slog.New(slog.DiscardHandler), + }) + require.NoError(t, err) + } + close(stop) + wg.Wait() +} diff --git a/internal/connector/setup/private_state.go b/internal/connector/setup/private_state.go new file mode 100644 index 000000000..81b55d319 --- /dev/null +++ b/internal/connector/setup/private_state.go @@ -0,0 +1,162 @@ +package setup + +import ( + "errors" + "fmt" + "os" + "path/filepath" + + "github.com/gofrs/flock" +) + +// ErrLockHeld reports a lock another process already holds. +var ErrLockHeld = errors.New("the lock is held by another process") + +// EnsurePrivateDir creates dir owner-only when it is missing, and refuses it +// when dir — or any directory above it — could be changed by someone else. +// +// It is the same check connect.json gets, exported for the state the running +// connector keeps beside it: the instance lock and the ledger live under the +// same config root, and a directory another local user can write is a +// directory where a lock file can be swapped for one that excludes nobody. +// +// Only the last component is created. Everything above it must already exist +// and already be trustworthy, which is what makes the refusal meaningful. +func EnsurePrivateDir(dir string) error { + if dir == "" { + return fmt.Errorf("%w: no directory given", ErrNotPrivate) + } + abs, err := filepath.Abs(dir) + if err != nil { + return err + } + if err := checkAncestors(filepath.Dir(abs)); err != nil { + return err + } + switch _, err := os.Lstat(abs); { + case errors.Is(err, os.ErrNotExist): + if err := os.Mkdir(abs, 0o700); err != nil && !errors.Is(err, os.ErrExist) { + return fmt.Errorf("create %s: %w", abs, err) + } + case err != nil: + return fmt.Errorf("inspect %s: %w", abs, err) + } + return checkPrivateDir(abs) +} + +// TryLockPrivate takes an exclusive lock on path without waiting, having first +// established that the lock is a lock: the directory holding it is this user's +// alone, and the file itself is this user's own regular file rather than a +// symlink or something another user left there. A lock on a file two processes +// can be pointed at different inodes excludes nobody. +// +// It reports ErrLockHeld when another process holds it, which is an answer +// rather than a failure. Every other error means the lock could not be +// established, and the caller must refuse rather than carry on unlocked. +func TryLockPrivate(path string) (unlock func() error, err error) { + if err := EnsurePrivateDir(filepath.Dir(path)); err != nil { + return nil, err + } + if err := checkPrivateLockFile(path); err != nil { + if errors.Is(err, ErrNotPrivate) { + return nil, err + } + return nil, fmt.Errorf("%w: %s: %w", ErrLockUnavailable, path, err) + } + lock := flock.New(path, flock.SetPermissions(0o600)) + held, err := lock.TryLock() + if err != nil { + return nil, fmt.Errorf("%w: %s: %w", ErrLockUnavailable, path, err) + } + if !held { + return nil, ErrLockHeld + } + // Re-checked with the lock held, and this time the file must be there: + // a lock file that has been unlinked or replaced since the open excludes + // nobody, whatever this process is holding. + if err := checkHeldLockFile(path); err != nil { + _ = lock.Unlock() + return nil, err + } + return lock.Unlock, nil +} + +// checkHeldLockFile is checkPrivateLockFile for a lock already taken, where a +// missing file is itself the tampering it is looking for. +func checkHeldLockFile(path string) error { + f, err := openNoFollow(path) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("%w: %s was removed while it was being locked", ErrNotPrivate, path) + } + return fmt.Errorf("%w: %s: %w", ErrLockUnavailable, path, err) + } + defer f.Close() + return checkPrivateFile(f, path) +} + +// EnsurePrivateFile creates path owner-only when it is missing, and refuses it +// when it is not this user's own regular file reached through directories only +// this user can change. +// +// It is stricter than the check connect.json gets in one way: a file another +// user can merely READ is refused too. It exists for the connector's ledger, +// which holds the account's feed positions — resumable tokens, so credentials +// — and there a readable file is already the leak. +// +// What is validated is the descriptor, not the name: the file is opened +// without following symlinks and inspected through that open file, so the +// thing checked is the thing the caller will go on to use. Creation is +// exclusive for the same reason — O_CREAT|O_EXCL refuses a symlink outright +// rather than following it somewhere else. +func EnsurePrivateFile(path string) error { + if err := EnsurePrivateDir(filepath.Dir(path)); err != nil { + return err + } + f, err := openNoFollow(path) + if errors.Is(err, os.ErrNotExist) { + created, createErr := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + switch { + case createErr == nil: + // The handle is writable, so its close is reported rather than + // deferred away: a close that fails is a file that may not be + // there, and the caller is about to treat it as one that is. + // Nothing has been written to it, so the durable part is the + // directory entry, which is what is synced. + if err := checkPrivateReadableFile(created, path); err != nil { + _ = created.Close() + return err + } + if err := created.Close(); err != nil { + return fmt.Errorf("create %s: %w", path, err) + } + syncDir(filepath.Dir(path)) + return nil + case !errors.Is(createErr, os.ErrExist): + return fmt.Errorf("create %s: %w", path, createErr) + } + // Something appeared between the open and the create — including, + // possibly, a symlink, which is why this reopen still refuses to + // follow one. + f, err = openNoFollow(path) + } + if err != nil { + return err + } + defer f.Close() + return checkPrivateReadableFile(f, path) +} + +func checkPrivateReadableFile(f *os.File, path string) error { + if err := checkPrivateFile(f, path); err != nil { + return err + } + info, err := f.Stat() + if err != nil { + return fmt.Errorf("inspect %s: %w", path, err) + } + if perm := info.Mode().Perm(); perm&0o077 != 0 { + return fmt.Errorf("%w: %s can be read by other users (mode %04o); it must be 0600", ErrNotPrivate, path, perm) + } + return nil +} diff --git a/internal/connector/shared_lines_test.go b/internal/connector/shared_lines_test.go new file mode 100644 index 000000000..efe0dec94 --- /dev/null +++ b/internal/connector/shared_lines_test.go @@ -0,0 +1,83 @@ +package connector + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/basecamp/basecamp-cli/internal/connector/admission" + "github.com/basecamp/basecamp-cli/internal/connector/ndjson" +) + +// A sink that is safe for concurrent use but takes three bytes at a time, so +// two producers that do not share a lock tear each other's lines. It is a +// value type, which is the case the writer registry cannot settle by itself. +type valueSink struct { + mu *sync.Mutex + buf *bytes.Buffer +} + +func (s valueSink) Write(p []byte) (int, error) { + s.mu.Lock() + defer s.mu.Unlock() + if len(p) > 3 { + p = p[:3] + } + return s.buf.Write(p) +} + +// Intake's pointer lines and admission's verdict lines share one stdout, so +// the composition hands them one writer. +func TestIntakeAndAdmissionShareOneLineWriter(t *testing.T) { + sink := valueSink{mu: &sync.Mutex{}, buf: &bytes.Buffer{}} + testSink = sink + lines := ndjson.NewWriter(sink) + + intake, _, _ := newTestIntakeLines(t, lines) + verdicts := &lineWriterFor{lines: lines} + + ctx := context.Background() + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + for id := int64(1); id <= 200; id++ { + require.NoError(t, intake.ingest(ctx, testEvent(id), LanePoll)) + } + }() + go func() { + defer wg.Done() + for id := int64(1); id <= 200; id++ { + require.NoError(t, verdicts.write(id)) + } + }() + wg.Wait() + + scanner := bufio.NewScanner(bytes.NewReader(sink.buf.Bytes())) + lineCount := 0 + for scanner.Scan() { + var v map[string]any + require.NoError(t, json.Unmarshal(scanner.Bytes(), &v), "torn line %d: %q", lineCount, scanner.Text()) + lineCount++ + } + assert.Equal(t, 400, lineCount) +} + +// lineWriterFor stands in for admission's verdict lines over the same writer. +type lineWriterFor struct{ lines *ndjson.Writer } + +func (l *lineWriterFor) write(id int64) error { + return l.lines.WriteLine(admission.Line{Type: "event", EventID: id, EventType: "comment.created"}) +} + +var testSink valueSink + +// linesSink is the sink the test's writer was built for, so a mutation can +// build a second writer over the same one. +func linesSink(*ndjson.Writer) valueSink { return testSink } diff --git a/internal/connector/shutdown.go b/internal/connector/shutdown.go new file mode 100644 index 000000000..1e9299256 --- /dev/null +++ b/internal/connector/shutdown.go @@ -0,0 +1,40 @@ +package connector + +import ( + "os" + "os/signal" + "syscall" +) + +// Exit codes for a signaled shutdown. 128+n is the shell's convention, and a +// supervisor reading the connector's exit status needs to tell an interrupt +// from a termination from a crash. +const ( + // ExitInterrupted is 128 + SIGINT. + ExitInterrupted = 130 + // ExitTerminated is 128 + SIGTERM. + ExitTerminated = 143 +) + +// ExitCodeForSignal maps a shutdown signal to the exit code the connector +// leaves behind. An unknown signal reports 1: the process ended, and pretending +// it ended cleanly would be a lie to whatever restarts it. +func ExitCodeForSignal(sig os.Signal) int { + switch sig { + case os.Interrupt, syscall.SIGINT: + return ExitInterrupted + case syscall.SIGTERM: + return ExitTerminated + default: + return 1 + } +} + +// NotifyShutdown returns a channel carrying the first shutdown signal, and a +// stop function. Separated from the exit-code mapping so the mapping can be +// tested without sending real signals to the test binary. +func NotifyShutdown() (<-chan os.Signal, func()) { + ch := make(chan os.Signal, 1) + signal.Notify(ch, os.Interrupt, syscall.SIGTERM) + return ch, func() { signal.Stop(ch) } +} diff --git a/internal/connector/shutdown_test.go b/internal/connector/shutdown_test.go new file mode 100644 index 000000000..90ab7662c --- /dev/null +++ b/internal/connector/shutdown_test.go @@ -0,0 +1,17 @@ +package connector + +import ( + "os" + "syscall" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestExitCodeForSignal(t *testing.T) { + assert.Equal(t, ExitInterrupted, ExitCodeForSignal(os.Interrupt)) + assert.Equal(t, ExitInterrupted, ExitCodeForSignal(syscall.SIGINT)) + assert.Equal(t, ExitTerminated, ExitCodeForSignal(syscall.SIGTERM)) + assert.Equal(t, 1, ExitCodeForSignal(syscall.SIGHUP), + "an unrecognized end is not a clean one") +} diff --git a/internal/connector/since_test.go b/internal/connector/since_test.go new file mode 100644 index 000000000..28edc0523 --- /dev/null +++ b/internal/connector/since_test.go @@ -0,0 +1,60 @@ +package connector + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" +) + +// A3: an explicit --since stays the entry until a checkpoint supersedes it. A +// connection that ends before its first page is saved — here, a reconnect for +// a project the live subscription does not hold, raised mid-page — must not +// leave the next connection with no position, entering at the present and +// skipping the history the operator asked for. +func TestAnExplicitSinceSurvivesAReconnectBeforeTheFirstCheckpoint(t *testing.T) { + ledger := newTestLedger(t) + intake, transport, minter, polls := newFeedIntake(t, ledger, Options{ + SinceEventID: 17099838000, + Membership: &flakyMembership{buckets: []int64{48699913}}, + MembershipInterval: time.Hour, + }) + minter.ScriptTicket(ticket()) + minter.ScriptTicket(ticket()) + + unlisted := testEvent(17099838001) + unlisted.BucketID = 777 + // The first event raises the reconnect; the second is then refused + // delivery, so the page is never checkpointed. + polls.ScriptPage(eventfeed.PollPage{Events: []eventfeed.Event{unlisted, testEvent(17099838002)}, Position: "never-saved"}) + polls.ScriptPage(eventfeed.PollPage{Events: []eventfeed.Event{testEvent(17099838002)}, Position: "saved"}) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := runInBackground(ctx, t, intake) + subscribedConn(t, transport) + + answered := 1 + require.Eventually(t, func() bool { + if conns := transport.Conns(); len(conns) > answered { + answerSubscription(conns[len(conns)-1]) + answered = len(conns) + } + return polls.CallCount() >= 2 + }, 5*time.Second, 10*time.Millisecond) + + assert.Equal(t, "17099838000", polls.Calls()[1].Cursor.Since, + "nothing was checkpointed, so the reconnect still enters where the operator asked") + + require.Eventually(t, func() bool { + _, ok, err := ledger.Get(ctx, 17099838002) + return err == nil && ok + }, 5*time.Second, 10*time.Millisecond, "no event after --since is skipped") + + cancel() + awaitReturn(t, done, "Run should return on shutdown") +} diff --git a/internal/mcpserver/model/PROVENANCE.json b/internal/mcpserver/model/PROVENANCE.json index 25c72af63..41423b4a7 100644 --- a/internal/mcpserver/model/PROVENANCE.json +++ b/internal/mcpserver/model/PROVENANCE.json @@ -1,7 +1,7 @@ { "source": "github.com/basecamp/basecamp-sdk", - "commit": "4523eac74cfa6452ec85ea5f8f633baa4dc487a5", - "ref": "go/v0.18.0-63-g4523eac7", + "commit": "6abe227204a465ced72969f10e32d3a06e77bcc1", + "ref": "go/v0.19.0", "files": ["behavior-model.json", "openapi.json"], "synced_by": "scripts/sync-mcp-model.sh", "patches": "binary-upload operations dropped because the toolkit refuses their non-JSON bodies (EXCLUDED_OPERATIONS) and the stream-ticket mint dropped by policy (POLICY_EXCLUDED_OPERATIONS); no tag patches applied (PATCHED_TAGS is empty — the export tags every operation) — see the sync script" diff --git a/internal/version/sdk-provenance.json b/internal/version/sdk-provenance.json index 3ff436f25..d25aac38f 100644 --- a/internal/version/sdk-provenance.json +++ b/internal/version/sdk-provenance.json @@ -1,9 +1,9 @@ { "sdk": { "module": "github.com/basecamp/basecamp-sdk/go", - "version": "v0.18.1-0.20260916210227-4523eac74cfa", - "revision": "4523eac74cfa", - "updated_at": "2026-09-16T21:02:27Z" + "version": "v0.19.0", + "revision": "6abe227204a4", + "updated_at": "2026-09-16T21:15:28Z" }, "api": { "repo": "basecamp/bc3", diff --git a/nix/package.nix b/nix/package.nix index c68146eaf..82e2a2522 100644 --- a/nix/package.nix +++ b/nix/package.nix @@ -8,7 +8,7 @@ buildGoModule.override { go = go_1_26; } (finalAttrs: { src = lib.cleanSource ./..; # To update: set to lib.fakeHash, run `nix build`, use the hash from the error. - vendorHash = "sha256-mT0oVMMRcmpbszb0zVYd1k8cZfMHKHZRbhsy/+8GHxo="; + vendorHash = "sha256-fbSMybSFUlSHIE8/aqLH/QCu+Q3cvJV/qpBAKz7VAZI="; subPackages = [ "cmd/basecamp" ];