fix: correct information schema views metadata

docs/rfcs/20260903_information_schema_views_definition_protocol.md

@@ -0,0 +1,82 @@
+- Status: proposed — implementation complete; pending independent approval
+- Start Date: 2026-09-03
+- Authors: MatrixOne maintainers
+- Implementation PR: https://github.com/matrixorigin/matrixone/pull/27716
+- Issue for this RFC: https://github.com/matrixorigin/matrixone/issues/27655
+
+# Parser-derived `information_schema.VIEWS` definitions
+
+## Summary
+
+`information_schema.VIEWS.VIEW_DEFINITION` must expose the defining SELECT,
+not the original CREATE statement. New views persist a parser-derived definition
+and legacy rows are read through parser-aware metadata functions. The functions
+are new distributed plan functions (IDs 579 and 580), so the catalog contract is fenced by MORPC
+v59.
+
+## Problem and invariant
+
+Schema-diff and migration clients replay `VIEW_DEFINITION`. A full CREATE
+statement is not a standalone SELECT and falsely marks aggregate views as
+updatable. The invariant is that every visible current or legacy view returns
+its parser-derived frozen SELECT (or NULL only for a malformed catalog row),
+and no CN that cannot resolve either function ID can receive a pipeline or catalog view
+that references it.
+
+## Design
+
+The CREATE/ALTER owner derives `ViewData.Definition` from the stabilized view
+AST, after wildcard expansion and separately persists `CheckOption`. The
+catalog remains the single owner of that frozen metadata.
+`mo_view_definition(viewdef)` and `mo_view_check_option(viewdef)` return the
+stored fields without writes; for an older row that lacks them, they parse only
+the stored statement using its persisted SQL mode and identifier-case settings.
+This bounded, side-effect-free fallback avoids a second SQL regexp lexer and
+does not depend on background recovery.
+
+MORPC v59 is allocated as `MORPCLatestVersion + 1` from official main v58,
+which is already assigned to binary-string function semantics and runtime-domain metadata. It is specific to
+this function and the persisted VIEWS definition. The v4.0.6 VIEWS upgrade
+waits for common v59. New tenant initialization at v58 or below installs the
+predecessor VIEWS DDL, which has no function reference; v59 installs the new
+DDL. Pipeline preparation, remote
+marshal, and remote unmarshal reject a pipeline containing either function ID
+below v59. The receiver check protects stale prepared work as well as normal
+sender dispatch. Before admitting any v58-or-earlier CN during rollback,
+operators must restore `InformationSchemaViewsLegacyDDL` and wait for that
+catalog change to converge; merely draining v59-dependent requests is not
+sufficient because the new persisted view text references the function. The
+new JSON fields are additive and old binaries keep treating them as unknown.
+
+## Alternatives
+
+Keeping raw SQL regexp extraction was rejected because it repeatedly diverged
+from the SQL lexer for comments and quoted strings. Eagerly rewriting every
+legacy row was rejected because the existing recovery lifecycle is deliberately
+inactive and a metadata read must not perform unbounded catalog writes. Allowing
+the DDL before v59 was rejected because an old CN cannot bind the metadata functions.
+
+## Bounds, security, and operations
+
+The compatibility parse is per visible legacy row and is linear in that row's
+stored statement; current rows return their stored definition directly. It
+creates no durable work, goroutine, queue, retry, or cache. Existing visibility
+joins remain the authorization boundary, so parsing happens only after the view
+row is selected. A mixed-version request fails before dispatch with a stable
+NotSupported error rather than returning wrong metadata.
+
+## Validation
+
+Focused parser/function tests cover current and legacy definitions, quoted and
+commented inputs, malformed rows, frozen wildcard expansion, and CHECK OPTION.
+Protocol tests cover the v58 predecessor rejection and v59 acceptance at
+prepare, sender, and receiver boundaries. System-view tests prove v58 tenant
+initialization uses the predecessor DDL and v59 uses the parser-derived DDL;
+upgrade tests prove the VIEWS entry requires v59. The predecessor-init test is
+also the rollback guard: it proves that the restoration target has no function
+reference before an older CN is admitted.
+
+## Unresolved questions
+
+None. This RFC is proposed pending independent design approval; it documents
+the delivery contract and does not self-approve the design.

pkg/bootstrap/versions/v4_0_6/tenant_upgrade_list.go

@@ -86,6 +86,10 @@ func upgradeInformationSchemaMetadataVisibilityView(viewName, viewDDL string) ve
 	requiredProtocol := defines.MORPCVersion41
 	if viewName == "TABLES" || viewName == "COLUMNS" {
 		requiredProtocol = defines.MORPCVersion46
+	} else if viewName == "VIEWS" {
+		// The definition functions are encoded into remotely executed plans. Do not
+		// install this catalog contract until every CN can resolve function IDs 579 and 580.
+		requiredProtocol = defines.MORPCVersion59
 	}
 	return versions.UpgradeEntry{
 		Schema:                  sysview.InformationDBConst,

pkg/bootstrap/versions/v4_0_6/upgrade_test.go

@@ -45,7 +45,7 @@ func TestColumnsUpgradeProtocolGenerations(t *testing.T) {
 		upgradeInformationSchemaColumnsBinaryStrings(),
 		refreshInformationSchemaCharacterSetsUTF8Maxlen(),
 	} {
-		for _, peer := range []int64{defines.MORPCVersion46, defines.MORPCVersion57, defines.MORPCVersion58} {
+		for _, peer := range []int64{defines.MORPCVersion46, defines.MORPCVersion57, defines.MORPCVersion58, defines.MORPCVersion59} {
 			t.Run(fmt.Sprintf("%s-gate-%d-peer-%d", entry.TableName, entry.RequiredProtocolVersion, peer), func(t *testing.T) {
 				mp := mpool.MustNewZero()
 				defer mpool.DeleteMPool(mp)
@@ -65,7 +65,8 @@ func TestColumnsUpgradeProtocolGenerations(t *testing.T) {
 				entry.CheckFunc = func(executor.TxnExecutor, uint32) (bool, error) { return false, nil }
 				err := entry.Upgrade(txn, 0)
 				if peer < entry.RequiredProtocolVersion {
-					require.ErrorContains(t, err, "requires all CNs to support protocol version 58")
+					require.ErrorContains(t, err, fmt.Sprintf(
+						"requires all CNs to support protocol version %d", entry.RequiredProtocolVersion))
 					require.Empty(t, executed, "an old peer must block before DROP/DELETE or DDL")
 				} else {
 					require.NoError(t, err)
@@ -216,7 +217,10 @@ func TestUpgradeEntries(t *testing.T) {
 		"drop view if exists information_schema.statistics")
 	for _, entry := range tenantUpgEntries {
 		ddl := entry.UpgSql + entry.PostSql
-		if strings.Contains(ddl, "mo_subscription_tables()") ||
+		if entry.TableName == "VIEWS" {
+			require.Equal(t, int64(defines.MORPCVersion59), entry.RequiredProtocolVersion,
+				"view upgrade %s must wait for mo_view_definition", entry.TableName)
+		} else if strings.Contains(ddl, "mo_subscription_tables()") ||
 			strings.Contains(ddl, "mo_subscription_columns()") {
 			require.GreaterOrEqual(t, entry.RequiredProtocolVersion, int64(defines.MORPCVersion46),
 				"view upgrade %s must wait for subscription metadata functions", entry.TableName)
@@ -257,6 +261,8 @@ func TestUpgradeEntries(t *testing.T) {
 		expectedProtocol := int64(defines.MORPCVersion41)
 		if view.name == "TABLES" || view.name == "COLUMNS" {
 			expectedProtocol = defines.MORPCVersion46
+		} else if view.name == "VIEWS" {
+			expectedProtocol = defines.MORPCVersion59
 		}
 		require.Equal(t, expectedProtocol, entry.RequiredProtocolVersion)
 		require.Contains(t, strings.ToLower(entry.PreSql),
@@ -471,6 +477,24 @@ func TestDaemonClaimPrecisionCheckUsesStoredType(t *testing.T) {
 	}
 }
 
+func TestInformationSchemaViewsUpgradeUsesLegacyDefinitionCompatibility(t *testing.T) {
+	// A pre-upgrade viewdef has Stmt but no parser-derived definition. The
+	// MODIFY_VIEW entry must keep the public metadata contract available through
+	// the parser-aware compatibility function instead of returning NULL while the
+	// separate lifecycle recovery remains inactive.
+	var viewsEntry *versions.UpgradeEntry
+	for i := range tenantUpgEntries {
+		if tenantUpgEntries[i].TableName == "VIEWS" {
+			viewsEntry = &tenantUpgEntries[i]
+			break
+		}
+	}
+	require.NotNil(t, viewsEntry)
+	require.Equal(t, versions.MODIFY_VIEW, viewsEntry.UpgType)
+	require.Contains(t, viewsEntry.UpgSql, "mo_view_definition(tbl.viewdef)")
+	require.NotContains(t, viewsEntry.UpgSql, "json_extract_string(tbl.viewdef, '$.definition')")
+}
+
 func TestInformationSchemaMetadataVisibilityUpgradeChecks(t *testing.T) {
 	views := []struct {
 		name string

pkg/defines/const.go

@@ -94,7 +94,8 @@ const (
 	MORPCVersion56     int64 = 56 // session-scoped AUTO_INCREMENT increment/offset and provenance
 	MORPCVersion57     int64 = 57 // Arrow LOAD external-scan pipeline payload
 	MORPCVersion58     int64 = 58 // binary-string function semantics and runtime-domain metadata
-	MORPCLatestVersion       = MORPCVersion58
+	MORPCVersion59     int64 = 59 // parser-derived information_schema.VIEWS definition function
+	MORPCLatestVersion       = MORPCVersion59
 )
 
 // DefaultLockWaitTimeoutSeconds is shared by the frontend default and by

pkg/queryservice/client/query_client_test.go

@@ -35,7 +35,7 @@ func TestMongoDBClientRetireRequiresProtocolVersion5(t *testing.T) {
 }
 
 func TestRefreshSessionAuthRequiresCurrentProtocolVersion(t *testing.T) {
-	assert.Equal(t, defines.MORPCVersion58, defines.MORPCLatestVersion)
+	assert.Equal(t, defines.MORPCVersion59, defines.MORPCLatestVersion)
 	assert.Equal(t, defines.MORPCVersion54, methodVersions[query.CmdMethod_RefreshSessionAuth])
 	assert.GreaterOrEqual(t, defines.MORPCLatestVersion, methodVersions[query.CmdMethod_RefreshSessionAuth])
 }

pkg/sql/compile/compile.go

@@ -7882,6 +7882,19 @@ func supportsRemotePadSpaceSemantics(service string) bool {
 	return ok && protocolVersion >= defines.MORPCVersion40
 }
 
+func supportsRemoteViewDefinitionFunction(service string) bool {
+	rt := moruntime.ServiceRuntime(service)
+	if rt == nil {
+		return false
+	}
+	version, ok := rt.GetGlobalVariables(moruntime.MOProtocolVersion)
+	if !ok {
+		return false
+	}
+	protocolVersion, ok := version.(int64)
+	return ok && protocolVersion >= defines.MORPCVersion59
+}
+
 func supportsRemoteParquetWholeFileFanout(service string) bool {
 	rt := moruntime.ServiceRuntime(service)
 	if rt == nil {

pkg/sql/compile/remote_expr.go

@@ -548,6 +548,12 @@ func pipelineContainsFunction(p *pipeline.Pipeline, predicate remoteFunctionPred
 	return containsFunctionInValue(reflect.ValueOf(p), nil, predicate)
 }
 
+func pipelineContainsFunctionID(p *pipeline.Pipeline, functionID int32) bool {
+	return pipelineContainsFunction(p, func(id, _ int32) bool {
+		return id == functionID
+	})
+}
+
 func containsFunctionInExpr(
 	expr *plan.Expr,
 	seen map[uintptr]struct{},

pkg/sql/compile/remote_expr_test.go

@@ -767,6 +767,99 @@ func TestPadSpaceRemoteProtocolValidationV40FastPathIsAllocationFree(t *testing.
 	require.Equal(t, float64(0), allocs)
 }
 
+func TestViewDefinitionRemoteProtocolValidationAtPrepareSendAndReceiveBoundaries(t *testing.T) {
+	proc := testutil.NewProcess(t)
+	rt := runtime.ServiceRuntime(proc.GetService())
+	previous, hadPrevious := rt.GetGlobalVariables(runtime.MOProtocolVersion)
+	t.Cleanup(func() {
+		if hadPrevious {
+			rt.SetGlobalVariables(runtime.MOProtocolVersion, previous)
+		} else {
+			rt.CompareAndDeleteGlobalVariables(runtime.MOProtocolVersion, defines.MORPCVersion59)
+		}
+	})
+
+	viewDefinitionType := types.T_text.ToType()
+	viewDefinition := &plan.Expr{
+		Typ: plan2.MakePlan2Type(&viewDefinitionType),
+		Expr: &plan.Expr_F{F: &plan.Function{
+			Func: &plan.ObjectRef{
+				Obj:     function.EncodeOverloadID(function.MO_VIEW_DEFINITION, 0),
+				ObjName: "mo_view_definition",
+			},
+			Args: []*plan.Expr{plan2.MakePlan2StringConstExprWithType("{}", false)},
+		}},
+	}
+	pipelineWithFunction := &pipeline.Pipeline{InstructionList: []*pipeline.Instruction{{
+		Op:          int32(vm.Projection),
+		ProjectList: []*plan.Expr{viewDefinition},
+	}}}
+	viewCheckOptionType := types.T_varchar.ToType()
+	viewCheckOption := &plan.Expr{
+		Typ: plan2.MakePlan2Type(&viewCheckOptionType),
+		Expr: &plan.Expr_F{F: &plan.Function{
+			Func: &plan.ObjectRef{
+				Obj:     function.EncodeOverloadID(function.MO_VIEW_CHECK_OPTION, 0),
+				ObjName: "mo_view_check_option",
+			},
+			Args: []*plan.Expr{plan2.MakePlan2StringConstExprWithType("{}", false)},
+		}},
+	}
+	pipelineWithCheckOption := &pipeline.Pipeline{InstructionList: []*pipeline.Instruction{{
+		Op:          int32(vm.Projection),
+		ProjectList: []*plan.Expr{viewCheckOption},
+	}}}
+
+	rt.SetGlobalVariables(runtime.MOProtocolVersion, defines.MORPCVersion59)
+	require.NoError(t, validateRemoteViewDefinitionPipelineProtocol(proc, pipelineWithFunction))
+	require.NoError(t, validateRemoteViewDefinitionPipelineProtocol(proc, pipelineWithCheckOption))
+
+	prepared := newScope(Remote)
+	prepared.Proc = proc
+	projection := projection.NewArgument()
+	projection.ProjectList = []*plan.Expr{viewDefinition}
+	prepared.setRootOperator(projection)
+	data, err := encodeRemoteScope(prepared, proc)
+	require.NoError(t, err)
+	_, err = encodeScope(prepared)
+	require.NoError(t, err)
+
+	// v58 is the immediate predecessor after the main-branch rebase; it
+	// supports the binary-string protocol addition but not these new function IDs.
+	rt.SetGlobalVariables(runtime.MOProtocolVersion, defines.MORPCVersion58)
+	require.NoError(t, validateRemoteViewDefinitionPipelineProtocol(proc, &pipeline.Pipeline{}))
+	require.ErrorContains(t, validateRemoteViewDefinitionPipelineProtocol(proc, pipelineWithFunction),
+		"requires MORPC protocol version 59")
+	require.ErrorContains(t, validateRemoteViewDefinitionPipelineProtocol(proc, pipelineWithCheckOption),
+		"requires MORPC protocol version 59")
+	_, err = encodeRemoteScope(prepared, proc)
+	require.ErrorContains(t, err, "requires MORPC protocol version 59")
+	_, err = encodeScope(prepared)
+	require.ErrorContains(t, err, "requires MORPC protocol version 59")
+	_, err = decodeScope(data, proc, true, nil)
+	require.ErrorContains(t, err, "requires MORPC protocol version 59")
+}
+
+func TestViewDefinitionRemoteProtocolValidationV59FastPathIsAllocationFree(t *testing.T) {
+	proc := testutil.NewProcess(t)
+	rt := runtime.ServiceRuntime(proc.GetService())
+	defer rt.SetGlobalVariables(runtime.MOProtocolVersion, defines.MORPCLatestVersion)
+	rt.SetGlobalVariables(runtime.MOProtocolVersion, defines.MORPCVersion59)
+
+	// A large ordinary pipeline makes an accidental reflective traversal visible.
+	ordinary := &pipeline.Pipeline{InstructionList: make([]*pipeline.Instruction, 1_000)}
+	for i := range ordinary.InstructionList {
+		ordinary.InstructionList[i] = &pipeline.Instruction{Op: int32(vm.Projection)}
+	}
+	require.NoError(t, validateRemoteViewDefinitionPipelineProtocol(proc, ordinary))
+	allocs := testing.AllocsPerRun(100, func() {
+		if err := validateRemoteViewDefinitionPipelineProtocol(proc, ordinary); err != nil {
+			panic(err)
+		}
+	})
+	require.Equal(t, float64(0), allocs)
+}
+
 func TestScopeContainsVarExprInAggArguments(t *testing.T) {
 	scope := newScope(Normal)
 	op := group.NewArgument()

pkg/sql/compile/remoterun.go

@@ -99,6 +99,9 @@ func encodeScope(s *Scope) ([]byte, error) {
 	if err = validateRemoteBinaryStringPipelineProtocol(s.Proc, p); err != nil {
 		return nil, err
 	}
+	if err = validateRemoteViewDefinitionPipelineProtocol(s.Proc, p); err != nil {
+		return nil, err
+	}
 	return p.Marshal()
 }
 
@@ -119,6 +122,9 @@ func encodeRemoteScope(s *Scope, proc *process.Process) ([]byte, error) {
 	if err = validateRemoteMongoUserQueryPipelineProtocol(proc, p); err != nil {
 		return nil, err
 	}
+	if err = validateRemoteViewDefinitionPipelineProtocol(proc, p); err != nil {
+		return nil, err
+	}
 	if err = validateRemoteParquetWholeFileFanoutPipelineProtocol(proc, p); err != nil {
 		return nil, err
 	}
@@ -225,6 +231,9 @@ func decodeScope(data []byte, proc *process.Process, isRemote bool, eng engine.E
 		if err = validateRemotePadSpacePipelineProtocol(proc, p); err != nil {
 			return nil, err
 		}
+		if err = validateRemoteViewDefinitionPipelineProtocol(proc, p); err != nil {
+			return nil, err
+		}
 		if err = validateRemoteParquetWholeFileFanoutPipelineProtocol(proc, p); err != nil {
 			return nil, err
 		}
@@ -2471,6 +2480,31 @@ func validateRemoteArrowLoadPipelineProtocol(proc *process.Process, p *pipeline.
 	return nil
 }
 
+// validateRemoteViewDefinitionPipelineProtocol protects the function IDs that
+// occur in the persisted VIEWS definition. It is used at both marshal and
+// unmarshal boundaries, so a stale prepared or remote pipeline fails closed
+// instead of being bound by a CN that predates the function registration.
+func validateRemoteViewDefinitionPipelineProtocol(
+	proc *process.Process,
+	p *pipeline.Pipeline,
+) error {
+	// A current peer cannot reject this function. Avoid a reflective traversal
+	// of every ordinary remote pipeline once the negotiated capability is known.
+	if proc != nil && supportsRemoteViewDefinitionFunction(proc.GetService()) {
+		return nil
+	}
+	if p == nil || (!pipelineContainsFunctionID(p, function.MO_VIEW_DEFINITION) &&
+		!pipelineContainsFunctionID(p, function.MO_VIEW_CHECK_OPTION)) {
+		return nil
+	}
+	if proc == nil || !supportsRemoteViewDefinitionFunction(proc.GetService()) {
+		return moerr.NewNotSupportedNoCtx(
+			"view metadata remote execution requires MORPC protocol version 59",
+		)
+	}
+	return nil
+}
+
 func aggregateUsesCollationAwareTextMinMax(agg aggexec.AggFuncExecExpression) bool {
 	if agg.GetAggID() != aggexec.AggIdOfMin && agg.GetAggID() != aggexec.AggIdOfMax {
 		return false