Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions docs/designs/knative-fiber-serving.md
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,18 @@ fresh-per-request and reuse execution, request completion, streams,
WebSockets, cancellation, and timeouts for the Revision already selected by
Knative routing.

### Revision route identities

The router uses `namespace/revision` as its pool key. The controller writes
`revision-<namespace>_<revision>.json` in the `lampyr-routes` ConfigMap.
Kubernetes namespace and Revision names cannot contain an underscore.

The router can still read the old filenames. When the controller publishes
a route, it removes the old `--` filename only if its document identifies
the same Revision. One ConfigMap update removes the old file and adds the
new file. The readiness check still uses the namespace, Revision name,
and document version. This change does not enable other namespaces.

## Revision backend feasibility gate

The upstream Route reconciler routes each traffic target to a Kubernetes Service
Expand Down
2 changes: 1 addition & 1 deletion internal/fiberdintegration/homes_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -740,7 +740,7 @@ func TestAPooledSlotLetsItsPrunedConnectionCloseWhenItEnds(t *testing.T) {
t.Fatal("a prune closed the connection a pooled slot holds")
}
router.mu.RLock()
pool := router.pools["default--demo"]
pool := router.pools["default/demo"]
router.mu.RUnlock()
router.endPool(pool, true, time.Now())
router.releases.Wait()
Expand Down
11 changes: 5 additions & 6 deletions internal/fiberdintegration/router.go
Original file line number Diff line number Diff line change
Expand Up @@ -294,12 +294,11 @@ func (router *Router) Reload() error {
return fmt.Errorf("duplicate Revision route %q", key)
}
revisions[key] = revision
poolKey := revision.Namespace + "--" + revision.Revision
if revision.Routed {
retain[poolKey] = true
retain[key] = true
routed[key] = true
execution, _ := (project.Manifest{Execution: &revision.Execution}).EffectiveExecution()
idle[poolKey], _ = time.ParseDuration(execution.IdleTimeout)
idle[key], _ = time.ParseDuration(execution.IdleTimeout)
}
}
router.mu.Lock()
Expand Down Expand Up @@ -608,7 +607,7 @@ func pause(request context.Context, deadline time.Time, d time.Duration) bool {
// measures what is left of that from when the Clone arrives, so a slow
// connect does not give the helper more time than the request has.
func (router *Router) clone(request context.Context, deadline time.Time, whole bool, namespace, revision string, execution project.ExecutionPolicy, grant HomeGrant) (consumer.Fiber, func(bool), error) {
uid := namespace + "--" + revision
poolKey := namespace + "/" + revision
if execution.Mode == project.ExecutionFresh {
cloneCtx, cancel := context.WithDeadline(request, deadline)
fiber, err := grant.Client.Clone(cloneCtx, grant.Token, "", time.Until(deadline), nil)
Expand All @@ -629,10 +628,10 @@ func (router *Router) clone(request context.Context, deadline time.Time, whole b
return consumer.Fiber{}, nil, err
}
router.mu.Lock()
pool := router.pools[uid]
pool := router.pools[poolKey]
if pool == nil {
pool = &routerPool{idle: idle}
router.pools[uid] = pool
router.pools[poolKey] = pool
}
router.mu.Unlock()
pool.mu.Lock()
Expand Down
76 changes: 73 additions & 3 deletions internal/fiberdintegration/router_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,76 @@ func revisionRequest(method, path string, revision RevisionRoute) *http.Request
return request
}

func TestRevisionPoolsKeepCompoundIdentitiesDistinct(t *testing.T) {
routes := t.TempDir()
revisions := []RevisionRoute{
{
Kind: "revision", Namespace: "a--b", Revision: "c",
Execution: project.ExecutionPolicy{Mode: project.ExecutionReuse, IdleTimeout: "30s"},
Timeout: "30s", Routed: true,
},
{
Kind: "revision", Namespace: "a", Revision: "b--c",
Execution: project.ExecutionPolicy{Mode: project.ExecutionReuse, IdleTimeout: "30s"},
Timeout: "30s", Routed: true,
},
}
for index, revision := range revisions {
data, err := json.Marshal(revision)
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(routes, fmt.Sprintf("route-%d.json", index)), data, 0o644); err != nil {
t.Fatal(err)
}
}
client := &testFiberClient{sessions: map[string]consumer.Fiber{}, servers: map[string]*httptest.Server{}}
router, err := NewRouter(testHomeResolver{client: client}, routes)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
router.Close()
for _, server := range client.servers {
server.Close()
}
for _, revision := range revisions {
forgetRevision(revision.Namespace, revision.Revision)
}
})
request := func(revision RevisionRoute, count int) {
t.Helper()
response := httptest.NewRecorder()
router.ServeHTTP(response, revisionRequest(http.MethodGet, "/test", revision))
if response.Code != http.StatusOK || response.Body.String() != fmt.Sprint(count) {
t.Errorf("%s/%s request = %d %q, want independent fiber count %d",
revision.Namespace, revision.Revision, response.Code, response.Body.String(), count)
}
}
for count := 1; count <= 2; count++ {
for _, revision := range revisions {
request(revision, count)
}
}
if len(router.pools) != 2 {
t.Errorf("pool count = %d, want 2", len(router.pools))
}
if err := os.Remove(filepath.Join(routes, "route-0.json")); err != nil {
t.Fatal(err)
}
if err := router.Reload(); err != nil {
t.Fatal(err)
}
request(revisions[1], 3)
router.releases.Wait()
if len(client.released) != 1 {
t.Errorf("removed Revision released %d fibers, want 1", len(client.released))
}
if len(router.pools) != 1 || client.next != 2 {
t.Errorf("remaining pools = %d, created fibers = %d, want 1 and 2", len(router.pools), client.next)
}
}

func TestFreshPolicyCreatesAndReleasesEveryRequest(t *testing.T) {
router, client, revision := routerFixture(t, project.ExecutionFresh)
for range 2 {
Expand Down Expand Up @@ -291,7 +361,7 @@ func TestReusePreservesLiveStateAndReleasesOnIdle(t *testing.T) {
if client.next != 1 || len(client.released) != 0 {
t.Fatalf("reused fiber was not retained: created=%d released=%d", client.next, len(client.released))
}
pool := router.pools["default--demo-1234"]
pool := router.pools["default/demo-1234"]
router.endPool(pool, false, time.Now().Add(time.Minute))
router.releases.Wait()
if len(client.released) != 1 || len(pool.slots) != 0 {
Expand All @@ -314,7 +384,7 @@ func TestBusyReusableFiberIsNotReleasedByIdleSweep(t *testing.T) {
if err != nil {
t.Fatal(err)
}
pool := router.pools["default--demo-1234"]
pool := router.pools["default/demo-1234"]
router.endPool(pool, false, time.Now().Add(time.Hour))
router.releases.Wait()
if len(client.released) != 0 {
Expand Down Expand Up @@ -407,7 +477,7 @@ func TestRevisionReloadUpdatesReuseIdleTimeout(t *testing.T) {
t.Fatal(err)
}
router.releases.Wait()
pool := router.pools["default--demo-1234"]
pool := router.pools["default/demo-1234"]
router.endPool(pool, false, time.Now().Add(2*time.Second))
router.releases.Wait()
if len(client.released) != 1 {
Expand Down
31 changes: 27 additions & 4 deletions internal/servingrevision/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -862,7 +862,24 @@ func ensureGrantIdentity(
}

func revisionMetadataKey(revision *servingv1.Revision) string {
return "revision-" + revision.Namespace + "--" + revision.Name + ".json"
return "revision-" + revision.Namespace + "_" + revision.Name + ".json"
}

func removeLegacyRevisionMetadata(data map[string]string, revision *servingv1.Revision) bool {
key := "revision-" + revision.Namespace + "--" + revision.Name + ".json"
var identity struct {
Kind string `json:"kind"`
Namespace string `json:"namespace"`
Revision string `json:"revision"`
}
if err := json.Unmarshal([]byte(data[key]), &identity); err != nil ||
identity.Kind != "revision" ||
identity.Namespace != revision.Namespace ||
identity.Revision != revision.Name {
return false
}
delete(data, key)
return true
}

func (r *Reconciler) publishRevision(
Expand Down Expand Up @@ -899,7 +916,10 @@ func (r *Reconciler) publishRevision(
config.Data = make(map[string]string)
}
key := revisionMetadataKey(revision)
if config.Data[key] == string(document) {
// Remove the owned legacy file in the same update, so the router
// never projects two files for this Revision during migration.
removedLegacy := removeLegacyRevisionMetadata(config.Data, revision)
if config.Data[key] == string(document) && !removedLegacy {
return nil
}
config.Data[key] = string(document)
Expand Down Expand Up @@ -960,10 +980,13 @@ func (r *Reconciler) FinalizeKind(ctx context.Context, revision *servingv1.Revis
if err != nil {
return err
}
if _, exists := config.Data[revisionMetadataKey(revision)]; !exists {
key := revisionMetadataKey(revision)
_, exists := config.Data[key]
removedLegacy := removeLegacyRevisionMetadata(config.Data, revision)
if !exists && !removedLegacy {
return nil
}
delete(config.Data, revisionMetadataKey(revision))
delete(config.Data, key)
_, err = configMaps.Update(ctx, config, metav1.UpdateOptions{})
return err
})
Expand Down
131 changes: 119 additions & 12 deletions internal/servingrevision/controller_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,16 +18,17 @@ import (
"fmt"
"io"
"math/big"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strconv"
"strings"
"testing"
"time"

"github.com/Azure/fiberd/pkg/tlsconf"
"github.com/google/go-containerregistry/pkg/name"
"github.com/hyperlight-dev/lampyr/internal/fiberdintegration"
"github.com/hyperlight-dev/lampyr/internal/project"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
Expand All @@ -36,6 +37,7 @@ import (
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/apimachinery/pkg/util/validation"
utilyaml "k8s.io/apimachinery/pkg/util/yaml"
dynamicfake "k8s.io/client-go/dynamic/fake"
"k8s.io/client-go/kubernetes/fake"
Expand Down Expand Up @@ -806,6 +808,101 @@ func TestValidateRevisionRequiresFiberdTemplate(t *testing.T) {
}
}

func TestRevisionRouterMetadataKeepsCompoundIdentitiesDistinct(t *testing.T) {
ctx := context.Background()
client := fake.NewSimpleClientset(&corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{Name: "lampyr-routes", Namespace: systemNamespace},
})
reconciler := &Reconciler{kube: client}
revisions := []*servingv1.Revision{
{ObjectMeta: metav1.ObjectMeta{Namespace: "a--b", Name: "c"}},
{ObjectMeta: metav1.ObjectMeta{Namespace: "a", Name: "b--c"}},
}
spec := revisionSpec{execution: project.ExecutionPolicy{Mode: project.ExecutionFresh}, timeout: time.Second}
for _, revision := range revisions {
if _, err := reconciler.publishRevision(ctx, revision, spec); err != nil {
t.Fatal(err)
}
key := revisionMetadataKey(revision)
if errors := validation.IsConfigMapKey(key); len(errors) != 0 {
t.Errorf("route filename %q is not a valid ConfigMap key: %v", key, errors)
}
}
config, err := client.CoreV1().ConfigMaps(systemNamespace).Get(ctx, "lampyr-routes", metav1.GetOptions{})
if err != nil {
t.Fatal(err)
}
if len(config.Data) != 2 {
t.Fatalf("published %d route files, want 2 independent Revisions: %v", len(config.Data), config.Data)
}
if err := reconciler.FinalizeKind(ctx, revisions[0]); err != nil {
t.Fatal(err)
}
config, err = client.CoreV1().ConfigMaps(systemNamespace).Get(ctx, "lampyr-routes", metav1.GetOptions{})
if err != nil {
t.Fatal(err)
}
if _, exists := config.Data[revisionMetadataKey(revisions[1])]; !exists {
t.Fatal("finalizing one Revision removed the other Revision's metadata")
}
}

func TestRevisionRouterMetadataMigratesLegacyFilename(t *testing.T) {
const legacyKey = "revision-a--b--c.json"
for _, operation := range []string{"publish", "republish", "finalize"} {
for _, legacy := range []struct {
name, document string
owned bool
}{
{"matching", `{"kind":"revision","namespace":"a--b","revision":"c"}`, true},
{"colliding", `{"kind":"revision","namespace":"a","revision":"b--c"}`, false},
{"invalid", `not-json`, false},
} {
t.Run(operation+"/"+legacy.name, func(t *testing.T) {
ctx := context.Background()
client := fake.NewSimpleClientset(&corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{Name: "lampyr-routes", Namespace: systemNamespace},
Data: map[string]string{legacyKey: legacy.document},
})
reconciler := &Reconciler{kube: client}
revision := &servingv1.Revision{ObjectMeta: metav1.ObjectMeta{Namespace: "a--b", Name: "c"}}
spec := revisionSpec{execution: project.ExecutionPolicy{Mode: project.ExecutionFresh}, timeout: time.Second}
if operation == "republish" {
if _, err := reconciler.publishRevision(ctx, revision, spec); err != nil {
t.Fatal(err)
}
config, err := client.CoreV1().ConfigMaps(systemNamespace).Get(ctx, "lampyr-routes", metav1.GetOptions{})
if err != nil {
t.Fatal(err)
}
config.Data[legacyKey] = legacy.document
if _, err := client.CoreV1().ConfigMaps(systemNamespace).Update(ctx, config, metav1.UpdateOptions{}); err != nil {
t.Fatal(err)
}
}
if operation != "finalize" {
if _, err := reconciler.publishRevision(ctx, revision, spec); err != nil {
t.Fatal(err)
}
} else if err := reconciler.FinalizeKind(ctx, revision); err != nil {
t.Fatal(err)
}
config, err := client.CoreV1().ConfigMaps(systemNamespace).Get(ctx, "lampyr-routes", metav1.GetOptions{})
if err != nil {
t.Fatal(err)
}
document, exists := config.Data[legacyKey]
if legacy.owned && exists {
t.Error("owned legacy filename survived migration")
}
if !legacy.owned && (!exists || document != legacy.document) {
t.Error("legacy metadata without a matching identity was changed")
}
})
}
}
}

func TestRevisionRouterMetadataLifecycle(t *testing.T) {
ctx := context.Background()
client := fake.NewSimpleClientset(&corev1.ConfigMap{
Expand Down Expand Up @@ -860,19 +957,29 @@ func TestRevisionRouterMetadataLifecycle(t *testing.T) {
!document.Routed {
t.Fatalf("router metadata = %#v", document)
}
router := httptest.NewServer(http.HandlerFunc(func(
response http.ResponseWriter,
request *http.Request,
) {
if request.URL.Query().Get("namespace") != revision.Namespace ||
request.URL.Query().Get("revision") != revision.Name ||
request.URL.Query().Get("version") != version {
http.Error(response, "unexpected projection", http.StatusServiceUnavailable)
}
}))
routes := t.TempDir()
legacyFile := filepath.Join(routes, "revision-default--demo-00001.json")
if err := os.WriteFile(legacyFile, []byte(config.Data[key]), 0o644); err != nil {
t.Fatal(err)
}
consumer, err := fiberdintegration.NewRouter(&fiberdintegration.CallerGrants{}, routes)
if err != nil {
t.Fatal(err)
}
defer consumer.Close()
router := httptest.NewServer(consumer)
defer router.Close()
reconciler.http = router.Client()
reconciler.router = router.URL
reconciler.router = router.URL + "/readyz"
if err := reconciler.waitForRouter(ctx, revision, version); err != nil {
t.Fatalf("legacy projection is not ready: %v", err)
}
if err := os.Rename(legacyFile, filepath.Join(routes, key)); err != nil {
t.Fatal(err)
}
if err := consumer.Reload(); err != nil {
t.Fatal(err)
}
if err := reconciler.waitForRouter(ctx, revision, version); err != nil {
t.Fatal(err)
}
Expand Down
Loading