From b74222f97f219e4c7e91e2ac34ac9e795cf86271 Mon Sep 17 00:00:00 2001 From: tobiadefami Date: Sat, 24 Oct 2026 15:40:42 +0100 Subject: [PATCH 1/3] INDIRECT phase 1: single-cell references, computed on demand INDIRECT resolves its text at evaluation time and records the cell it reads as a runtime dependency in the graph, so it is not volatile. When the target is not current yet in the running pass, the evaluation is deferred: the sort computes the target first, with an explicit stack, and then retries the formula. Covers A1 references to a single cell, on the same or another sheet. Co-Authored-By: Claude Opus 5.5 --- .../src/DependencyGraph/DependencyGraph.ts | 182 ++++++++++++++- .../src/DependencyGraph/FormulaVertex.ts | 29 +++ hyperformula/src/DependencyGraph/Graph.ts | 44 +++- hyperformula/src/DependencyGraph/TopSort.ts | 217 ++++++++++++++++-- hyperformula/src/EvaluationPass.ts | 83 +++++++ hyperformula/src/Evaluator.ts | 152 +++++++++--- hyperformula/src/GraphBuilder.ts | 6 + hyperformula/src/Operations.ts | 2 + hyperformula/src/error-message.ts | 1 + hyperformula/src/i18n/languages/enGB.ts | 1 + .../src/interpreter/EvaluationDeferred.ts | 20 ++ hyperformula/src/interpreter/Interpreter.ts | 6 + .../categories/lookup-and-reference.ts | 7 + .../src/interpreter/plugin/IndirectPlugin.ts | 97 ++++++++ hyperformula/src/interpreter/plugin/index.ts | 1 + .../src/license/functionCapabilities.ts | 2 +- 16 files changed, 776 insertions(+), 74 deletions(-) create mode 100644 hyperformula/src/EvaluationPass.ts create mode 100644 hyperformula/src/interpreter/EvaluationDeferred.ts create mode 100644 hyperformula/src/interpreter/plugin/IndirectPlugin.ts diff --git a/hyperformula/src/DependencyGraph/DependencyGraph.ts b/hyperformula/src/DependencyGraph/DependencyGraph.ts index 962dedff1a..01299a0ad0 100644 --- a/hyperformula/src/DependencyGraph/DependencyGraph.ts +++ b/hyperformula/src/DependencyGraph/DependencyGraph.ts @@ -6,7 +6,7 @@ import {AbsoluteCellRange, SimpleCellRange, simpleCellRange} from '../AbsoluteCellRange' import {absolutizeDependencies} from '../absolutizeDependencies' import {ArraySize} from '../ArraySize' -import {CellError, ErrorType, isSimpleCellAddress, simpleCellAddress, SimpleCellAddress} from '../Cell' +import {addressKey, CellError, ErrorType, isSimpleCellAddress, simpleCellAddress, SimpleCellAddress} from '../Cell' import {RawCellContent} from '../CellContentParser' import {CellDependency} from '../CellDependency' import {Config} from '../Config' @@ -40,7 +40,7 @@ import { import {AddressMapping} from './AddressMapping/AddressMapping' import {ArrayMapping} from './ArrayMapping' import {collectAddressesDependentToRange} from './collectAddressesDependentToRange' -import {FormulaVertex} from './FormulaVertex' +import {FormulaVertex, RuntimeDependency} from './FormulaVertex' import {DependencyQuery, Graph} from './Graph' import {RangeMapping} from './RangeMapping' import {SheetMapping} from './SheetMapping' @@ -53,6 +53,11 @@ export class DependencyGraph { public readonly graph: Graph private changes: ContentChanges = ContentChanges.empty() public readonly sheetReferenceRegistrar: SheetReferenceRegistrar + /** + * Number of runtime dependency edges added to the graph so far. Lets the evaluator check that each pass that defers + * a formula makes progress. + */ + public runtimeEdgesAdded = 0 constructor( public readonly addressMapping: AddressMapping, @@ -263,6 +268,86 @@ export class DependencyGraph { return { vertex: newVertex, id: newVertexId } } + /** + * Adds an edge from the cell at the given address to a formula that read it during evaluation. + * Creates an empty cell vertex if there is none at the address. + * + * @returns the vertex of the cell that was read + */ + public addRuntimeDependency(dependent: FormulaVertex, address: SimpleCellAddress): CellVertex { + const { vertex, id } = this.fetchCellOrCreateEmpty(address) + if (!this.graph.existsEdge(vertex, dependent)) { + this.graph.addEdge(id ?? vertex, dependent) + this.runtimeEdgesAdded++ + } + return vertex + } + + /** + * Stores the runtime dependencies recorded in a completed evaluation of a formula. + * Removes the edges of stored runtime dependencies that were not read this time, + * unless the formula also depends on that cell through its AST. + */ + public commitRuntimeDependencies(dependent: FormulaVertex, recorded: RuntimeDependency[]): void { + if (recorded.length === 0 && dependent.runtimeDependencies === undefined) { + return + } + const stored = dependent.runtimeDependencies ?? [] + const recordedVertices = new Set(recorded.map(dependency => dependency.vertex)) + const stale = stored.filter(dependency => this.graph.hasNode(dependency.vertex) && !recordedVertices.has(dependency.vertex)) + + if (stale.length > 0) { + const staticVertices = this.staticDependencyVertices(dependent) + stale.forEach((dependency) => { + if (!staticVertices.has(dependency.vertex)) { + this.removeRuntimeEdge(dependency, dependent) + } + }) + } + + dependent.runtimeDependencies = recorded.length > 0 ? recorded : undefined + } + + /** + * Adds the runtime dependencies recorded in an interrupted evaluation of a formula to the stored ones, removing none. + */ + public keepRuntimeDependencies(dependent: FormulaVertex, recorded: RuntimeDependency[]): void { + if (recorded.length === 0) { + return + } + const stored = (dependent.runtimeDependencies ?? []).filter(dependency => this.graph.hasNode(dependency.vertex)) + const storedVertices = new Set(stored.map(dependency => dependency.vertex)) + dependent.runtimeDependencies = [...stored, ...recorded.filter(dependency => !storedVertices.has(dependency.vertex))] + } + + /** + * Removes all runtime dependencies of a formula and their edges, except edges the formula also has through its AST. + * + * @returns true if any edge was removed + */ + public clearRuntimeDependencies(dependent: FormulaVertex): boolean { + const stored = dependent.runtimeDependencies + dependent.runtimeDependencies = undefined + if (stored === undefined) { + return false + } + + const live = stored.filter(dependency => this.graph.hasNode(dependency.vertex)) + if (live.length === 0) { + return false + } + + const staticVertices = this.staticDependencyVertices(dependent) + let removedAny = false + live.forEach((dependency) => { + if (!staticVertices.has(dependency.vertex)) { + this.removeRuntimeEdge(dependency, dependent) + removedAny = true + } + }) + return removedAny + } + public removeRows(removedRows: RowsSpan): EagerChangesGraphChangeResult { this.stats.measure(StatType.ADJUSTING_GRAPH, () => { for (const [address, vertex] of this.addressMapping.entriesFromRowsSpan(removedRows)) { @@ -316,6 +401,16 @@ export class DependencyGraph { this.stats.measure(StatType.ADJUSTING_RANGES, () => { this.markAllRangesAsDirtyInSheet(sheetId) }) + + // a formula like INDIRECT("NewSheet!A1") may now resolve + this.addStructuralNodesToChangeSet() + } + + /** + * Marks the formulas that depend on the sheet structure as dirty, e.g., after a sheet is renamed. + */ + public markStructureDependentFormulasAsDirty(): void { + this.addStructuralNodesToChangeSet() } /** @@ -512,6 +607,7 @@ export class DependencyGraph { for (const adjacentNode of this.graph.adjacentNodes(targetVertex)) { sourceVertex = sourceVertex ?? this.fetchCellOrCreateEmpty(targetAddress).vertex this.graph.addEdge(sourceVertex, adjacentNode) + this.replaceRuntimeDependencyVertex(adjacentNode, targetVertex, sourceVertex) this.graph.markNodeAsDirty(sourceVertex) } this.removeVertex(targetVertex) @@ -695,7 +791,7 @@ export class DependencyGraph { const dependenciesResult = this.formulaDependencyQuery(vertex) if (dependenciesResult !== undefined) { const [address, dependencies] = dependenciesResult - return dependencies.map((dependency: CellDependency) => { + const addresses: (SimpleCellAddress | SimpleCellRange)[] = dependencies.map((dependency: CellDependency) => { if (dependency instanceof NamedExpressionDependency) { return this.namedExpressions.namedExpressionOrPlaceholder(dependency.name, address.sheet).address } else if (isSimpleCellAddress(dependency)) { @@ -704,6 +800,7 @@ export class DependencyGraph { return simpleCellRange(dependency.start, dependency.end) } }) + return this.appendRuntimeDependencyAddresses(vertex as FormulaVertex, addresses) } else { return [] } @@ -867,6 +964,7 @@ export class DependencyGraph { adjNodesStored.forEach((adjacentNode) => { if (this.graph.hasNode(adjacentNode)) { this.graph.addEdge(newNode, adjacentNode) + this.replaceRuntimeDependencyVertex(adjacentNode, oldNode, newNode) } }) } @@ -910,7 +1008,7 @@ export class DependencyGraph { const dependenciesResult = this.formulaDependencyQuery(vertex) if (dependenciesResult !== undefined) { const [address, dependencies] = dependenciesResult - return dependencies.map((dependency: CellDependency) => { + const result: [SimpleCellAddress | SimpleCellRange, Vertex][] = dependencies.map((dependency: CellDependency) => { if (dependency instanceof AbsoluteCellRange) { return [dependency.start, this.rangeMapping.getVertexOrThrow(dependency.start, dependency.end)] } else if (dependency instanceof NamedExpressionDependency) { @@ -920,6 +1018,12 @@ export class DependencyGraph { return [dependency, this.addressMapping.getCellOrThrow(dependency)] } }) + for (const runtimeDependency of (vertex as FormulaVertex).runtimeDependencies ?? []) { + if (this.graph.hasNode(runtimeDependency.vertex)) { + result.push([runtimeDependency.address, runtimeDependency.vertex]) + } + } + return result } else { return [] } @@ -1280,6 +1384,7 @@ export class DependencyGraph { dependents.forEach((adjacentNode) => { if (this.graph.hasNode(adjacentNode)) { this.graph.addEdge(target, adjacentNode) + this.replaceRuntimeDependencyVertex(adjacentNode, source, target) } }) @@ -1315,6 +1420,10 @@ export class DependencyGraph { dependencies.delete(dependency) const [address, vertex] = dependency if (this.graph.hasNode(vertex) && this.graph.adjacentNodesCount(vertex) === 0) { + if (vertex instanceof EmptyCellVertex && isSimpleCellAddress(address) && this.addressMapping.getCell(address) !== vertex) { + // a runtime dependency whose address no longer maps to this vertex + continue + } if (vertex instanceof RangeVertex || vertex instanceof EmptyCellVertex) { this.graph.removeNode(vertex).forEach((candidate) => dependencies.add(candidate)) } @@ -1331,6 +1440,71 @@ export class DependencyGraph { this.cleanupPlaceholderSheets(affectedSheets) } + /** + * Appends the addresses of a formula's runtime dependencies to the addresses collected from its AST, skipping duplicates. + */ + private appendRuntimeDependencyAddresses(vertex: FormulaVertex, addresses: (SimpleCellAddress | SimpleCellRange)[]): (SimpleCellAddress | SimpleCellRange)[] { + const runtimeDependencies = vertex.runtimeDependencies + if (runtimeDependencies === undefined) { + return addresses + } + const keys = new Set(addresses.map(address => isSimpleCellAddress(address) ? addressKey(address) : undefined)) + for (const {address, vertex: dependencyVertex} of runtimeDependencies) { + if (this.graph.hasNode(dependencyVertex) && !keys.has(addressKey(address))) { + keys.add(addressKey(address)) + addresses.push(address) + } + } + return addresses + } + + /** + * Returns the cell vertices a formula depends on directly through its AST (cell references and named expressions). + */ + private staticDependencyVertices(vertex: FormulaVertex): Set { + const result = new Set() + const dependenciesResult = this.formulaDependencyQuery(vertex) + if (dependenciesResult === undefined) { + return result + } + const [formulaAddress, dependencies] = dependenciesResult + for (const dependency of dependencies) { + if (dependency instanceof AbsoluteCellRange) { + continue + } + const address = dependency instanceof NamedExpressionDependency + ? this.namedExpressions.namedExpressionOrPlaceholder(dependency.name, formulaAddress.sheet).address + : dependency + const cell = this.addressMapping.getCell(address) + if (cell !== undefined) { + result.add(cell) + } + } + return result + } + + /** + * Removes the edge of a runtime dependency. Removes the read cell's vertex too if it is an empty cell nothing else depends on. + */ + private removeRuntimeEdge(dependency: RuntimeDependency, dependent: FormulaVertex): void { + const {address, vertex} = dependency + this.graph.removeEdgeIfExists(vertex, dependent) + if (vertex instanceof EmptyCellVertex && this.graph.adjacentNodesCount(vertex) === 0 && this.addressMapping.getCell(address) === vertex) { + this.graph.removeNode(vertex) + this.addressMapping.removeCell(address) + this.cleanupPlaceholderSheets(new Set([address.sheet])) + } + } + + /** + * Points a formula's runtime dependencies on oldVertex to newVertex. + */ + private replaceRuntimeDependencyVertex(dependent: Vertex, oldVertex: Vertex, newVertex: Vertex): void { + if (dependent instanceof FormulaVertex && oldVertex instanceof CellVertex && newVertex instanceof CellVertex) { + dependent.replaceRuntimeDependencyVertex(oldVertex, newVertex) + } + } + /** * Removes placeholder sheets that have no remaining vertices. */ diff --git a/hyperformula/src/DependencyGraph/FormulaVertex.ts b/hyperformula/src/DependencyGraph/FormulaVertex.ts index 5aa448ffd8..3ad8431743 100644 --- a/hyperformula/src/DependencyGraph/FormulaVertex.ts +++ b/hyperformula/src/DependencyGraph/FormulaVertex.ts @@ -16,6 +16,16 @@ import {Ast} from '../parser' import {ColumnsSpan, RowsSpan} from '../Span' import {CellVertex} from './CellVertex' +/** + * A dependency discovered while evaluating a formula (e.g., the target of INDIRECT), as opposed to one collected + * from the formula's AST. + */ +export interface RuntimeDependency { + /** Address read by the formula. Informational: the vertex is the source of truth. */ + address: SimpleCellAddress, + /** Vertex that was read. The graph has an edge from it to the formula vertex. */ + vertex: CellVertex, +} /** * Abstract base class for vertices that contain formulas in the dependency graph. @@ -25,6 +35,11 @@ import {CellVertex} from './CellVertex' * and {@link ArrayFormulaVertex} for array formulas that span multiple cells. */ export abstract class FormulaVertex extends CellVertex { + /** + * Dependencies recorded during the last evaluation of the formula. Undefined for formulas that have none. + */ + public runtimeDependencies?: RuntimeDependency[] + protected constructor( protected formula: Ast, protected cellAddress: SimpleCellAddress, @@ -87,6 +102,20 @@ export abstract class FormulaVertex extends CellVertex { public abstract valueOrUndef(): Maybe public abstract isComputed(): boolean + + /** + * Points runtime dependencies on oldVertex to newVertex, when a cell vertex is replaced by another one. + */ + public replaceRuntimeDependencyVertex(oldVertex: CellVertex, newVertex: CellVertex): void { + if (this.runtimeDependencies === undefined) { + return + } + for (const dependency of this.runtimeDependencies) { + if (dependency.vertex === oldVertex) { + dependency.vertex = newVertex + } + } + } } /** diff --git a/hyperformula/src/DependencyGraph/Graph.ts b/hyperformula/src/DependencyGraph/Graph.ts index 0a4086dcff..7dc8add37b 100644 --- a/hyperformula/src/DependencyGraph/Graph.ts +++ b/hyperformula/src/DependencyGraph/Graph.ts @@ -5,7 +5,7 @@ import { SimpleCellAddress } from '../Cell' import { SimpleCellRange } from '../AbsoluteCellRange' -import { TopSort, TopSortResult } from './TopSort' +import { NodeProcessingResult, TopSort, TopSortResult } from './TopSort' import { ProcessableValue } from './ProcessableValue' export type NodeId = number @@ -15,6 +15,21 @@ export type DependencyQuery = (vertex: Node) => [(SimpleCellAddress | Simp * Interface for nodes that can be stored in the Graph. * The idInGraph property is managed by the Graph class and should not be set externally. */ +/** + * View of a running top-sort pass, passed to the operating function together with the node. + */ +export interface TopSortPass { + /** + * Returns true if the node was reached by the pass and its value may still change later in it. + */ + isPending(node: Node): boolean, + /** + * Requests that a pending node is computed before the node being processed is retried. + * Call it right before returning {@link NodeProcessingResult.DEFERRED}. + */ + requestPull(node: Node): void, +} + export interface GraphNode { idInGraph?: NodeId, } @@ -258,24 +273,41 @@ export class Graph { * Sorts the whole graph topologically. Nodes that are on cycles are kept separate. */ public topSortWithScc(): TopSortResult { - return this.getTopSortedWithSccSubgraphFrom(this.getNodes(), () => true, () => { }) + return this.getTopSortedWithSccSubgraphFrom(this.getNodes(), () => NodeProcessingResult.CHANGED, () => { }) } /** * Sorts the graph topologically. Nodes that are on cycles are kept separate. * * @param {Node[]} modifiedNodes - seed for computation. The algorithm assumes that only these nodes have changed since the last run. - * @param {(node: Node) => boolean} operatingFunction - recomputes value of a node, and returns whether a change occurred + * @param {(node: Node, pass: TopSortPass) => NodeProcessingResult} operatingFunction - recomputes value of a node, and returns the outcome * @param {(node: Node) => void} onCycle - action to be performed when node is on cycle */ public getTopSortedWithSccSubgraphFrom( modifiedNodes: Node[], - operatingFunction: (node: Node) => boolean, + operatingFunction: (node: Node, pass: TopSortPass) => NodeProcessingResult, onCycle: (node: Node) => void ): TopSortResult { - const topSortAlgorithm = new TopSort(this.nodesSparseArray, this.edgesSparseArray) + const precedentIds = (id: NodeId): NodeId[] => { + const node = this.nodesSparseArray[id] + if (node === undefined) { + return [] + } + return this.dependencyQuery(node) + .map(([, dependency]) => dependency.idInGraph) + .filter((dependencyId): dependencyId is NodeId => dependencyId !== undefined) + } + const topSortAlgorithm = new TopSort(this.nodesSparseArray, this.edgesSparseArray, precedentIds) + const pass: TopSortPass = { + isPending: (node: Node) => node.idInGraph !== undefined && topSortAlgorithm.isPending(node.idInGraph), + requestPull: (node: Node) => { + if (node.idInGraph !== undefined) { + topSortAlgorithm.requestPull(node.idInGraph) + } + }, + } const modifiedNodesIds = modifiedNodes.map(node => node.idInGraph).filter(id => id !== undefined) as NodeId[] - return topSortAlgorithm.getTopSortedWithSccSubgraphFrom(modifiedNodesIds, operatingFunction, onCycle) + return topSortAlgorithm.getTopSortedWithSccSubgraphFrom(modifiedNodesIds, (node: Node) => operatingFunction(node, pass), onCycle) } /** diff --git a/hyperformula/src/DependencyGraph/TopSort.ts b/hyperformula/src/DependencyGraph/TopSort.ts index b79e450fce..cdeb9ec644 100644 --- a/hyperformula/src/DependencyGraph/TopSort.ts +++ b/hyperformula/src/DependencyGraph/TopSort.ts @@ -6,6 +6,26 @@ export interface TopSortResult { sorted: T[], cycled: T[], + /** + * Nodes that still have to be recomputed in another pass. Always empty: a node that reads a pending node is + * retried in the same pass, after the pending node is computed. + */ + deferred: T[], +} + +/** + * Outcome of processing a single node during a top-sort pass. + */ +export enum NodeProcessingResult { + /** The node was recomputed and its value changed. */ + CHANGED = 'CHANGED', + /** The node was recomputed and its value did not change. */ + UNCHANGED = 'UNCHANGED', + /** + * The node read a value that is not current yet, and requested that node with {@link TopSort.requestPull}. + * It is recomputed again once the requested node is computed. + */ + DEFERRED = 'DEFERRED', } // node status life cycle: undefined -> ON_STACK -> PROCESSED -> POPPED @@ -26,11 +46,24 @@ export class TopSort { private nodeStatus: NodeVisitStatus[] = [] private order: number[] = [] private sccNonSingletons: boolean[] = [] + private finalized: boolean[] = [] + private onPullPath: boolean[] = [] + private pendingPrecedents: Map = new Map() + private pullRequest?: number + private shouldBeUpdatedMapping: boolean[] = [] + private sorted: T[] = [] + private cycled: T[] = [] private timeCounter: number = 0 + /** + * @param nodesSparseArray - nodes by id + * @param edgesSparseArray - ids of the dependents of each node, by id + * @param precedentIds - ids of the nodes a node reads; used only for nodes pulled out of order + */ constructor( private nodesSparseArray: T[] = [], private edgesSparseArray: number[][] = [], + private precedentIds: (id: number) => number[] = () => [], ) { } @@ -39,12 +72,12 @@ export class TopSort { * Returns vertices in order of topological sort, but vertices that are on cycles are kept separate. * * @param modifiedNodes - seed for computation. During engine init run, all of the vertices of grap. In recomputation run, changed vertices. - * @param operatingFunction - recomputes value of a node, and returns whether a change occurred + * @param operatingFunction - recomputes value of a node, and returns the outcome * @param onCycle - action to be performed when node is on cycle */ public getTopSortedWithSccSubgraphFrom( modifiedNodeIds: number[], - operatingFunction: (node: T) => boolean, + operatingFunction: (node: T) => NodeProcessingResult, onCycle: (node: T) => void ): TopSortResult { const modifiedNodeIdsReversed = modifiedNodeIds.reverse() @@ -52,6 +85,22 @@ export class TopSort { return this.postprocess(modifiedNodeIdsReversed, onCycle, operatingFunction) } + /** + * Returns true if the node was reached by this pass and has not been finalized yet, + * i.e., its value may still change later in the pass. + */ + public isPending(id: number): boolean { + return Boolean(this.inSCC[id]) && !this.finalized[id] + } + + /** + * Requests that a pending node is computed before the node being processed is retried. + * The operating function calls it right before it returns {@link NodeProcessingResult.DEFERRED}. + */ + public requestPull(id: number): void { + this.pullRequest = id + } + /** * Returns adjacent nodes of a given node. */ @@ -153,37 +202,155 @@ export class TopSort { } /** - * Postprocesses the result of Tarjan's algorithm. + * Postprocesses the result of Tarjan's algorithm: processes the nodes in topological order. + * + * A node whose operating function returns DEFERRED has read a pending node P. P is pulled: its pending precedents + * and then P itself are processed right away, from an explicit stack, and the node is retried. The main walk skips + * nodes that were already processed this way. A pull that reaches a node already on the pull path is a cycle. */ - private postprocess(modifiedNodeIds: number[], onCycle: (node: T) => void, operatingFunction: (node: T) => boolean) { - const shouldBeUpdatedMapping: boolean[] = [] - + private postprocess(modifiedNodeIds: number[], onCycle: (node: T) => void, operatingFunction: (node: T) => NodeProcessingResult): TopSortResult { modifiedNodeIds.forEach((t: number) => { - shouldBeUpdatedMapping[t] = true + this.shouldBeUpdatedMapping[t] = true }) - const sorted: T[] = [] - const cycled: T[] = [] this.order.reverse() - this.order.forEach((t: number) => { - const adjacentNodes = this.getAdjacentNodeIds(t) - - // The following line is a potential performance bottleneck. - // Array.includes() is O(n) operation, which makes the whole algorithm O(n^2). - // Idea for improvement: use Set[] instead of number[][] for edgesSparseArray. - if (this.sccNonSingletons[t] || adjacentNodes.includes(t)) { - cycled.push(this.nodesSparseArray[t]) - onCycle(this.nodesSparseArray[t]) - adjacentNodes.forEach((s: number) => shouldBeUpdatedMapping[s] = true) - } else { - sorted.push(this.nodesSparseArray[t]) - if (shouldBeUpdatedMapping[t] && operatingFunction(this.nodesSparseArray[t])) { - adjacentNodes.forEach((s: number) => shouldBeUpdatedMapping[s] = true) + for (const t of this.order) { + if (this.finalized[t] || this.nodesSparseArray[t] === undefined) { + continue + } + this.pullRequest = undefined + if (this.processNode(t, onCycle, operatingFunction) === NodeProcessingResult.DEFERRED) { + this.pullFor(t, onCycle, operatingFunction) + } + } + + return {sorted: this.sorted, cycled: this.cycled, deferred: []} + } + + /** + * Called after the root node was deferred: pulls the pending nodes it reads and retries it until it is finalized. + */ + private pullFor(root: number, onCycle: (node: T) => void, operatingFunction: (node: T) => NodeProcessingResult): void { + const stack: number[] = [root] + this.onPullPath[root] = true + this.pushRequestedNode(stack, onCycle) + + while (stack.length > 0) { + const t = stack[stack.length - 1] + + if (this.finalized[t] || this.nodesSparseArray[t] === undefined) { + stack.pop() + this.onPullPath[t] = false + continue + } + + if (t !== root) { + const precedent = this.nextPendingPrecedent(t) + if (precedent !== undefined) { + if (this.onPullPath[precedent]) { + this.finalizeCycle(stack, precedent, onCycle) + } else { + stack.push(precedent) + this.onPullPath[precedent] = true + } + continue } } - }) - return {sorted, cycled} + this.pullRequest = undefined + if (this.processNode(t, onCycle, operatingFunction) === NodeProcessingResult.DEFERRED) { + this.pushRequestedNode(stack, onCycle) + } + } + } + + /** + * Pushes the node requested by the deferred node on top of the stack, or gives the cycle treatment to the pull path + * if the requested node is already on it. + */ + private pushRequestedNode(stack: number[], onCycle: (node: T) => void): void { + const target = this.pullRequest + if (target === undefined || !this.isPending(target)) { + throw new Error('A node was deferred without requesting a pending node.') + } + if (this.onPullPath[target]) { + this.finalizeCycle(stack, target, onCycle) + } else { + stack.push(target) + this.onPullPath[target] = true + } + } + + /** + * Returns a precedent of a pulled node that is still pending, or undefined if there is none. + * The pending precedents are collected once per node and consumed as they get finalized. + */ + private nextPendingPrecedent(t: number): number | undefined { + let pending = this.pendingPrecedents.get(t) + if (pending === undefined) { + pending = this.precedentIds(t).filter(id => this.isPending(id)) + this.pendingPrecedents.set(t, pending) + } + while (pending.length > 0 && !this.isPending(pending[pending.length - 1])) { + pending.pop() + } + return pending.length > 0 ? pending[pending.length - 1] : undefined + } + + /** + * Gives every node on the pull path from the given node to the top of the stack the cycle treatment. + */ + private finalizeCycle(stack: number[], from: number, onCycle: (node: T) => void): void { + for (let i = stack.indexOf(from); i < stack.length; i++) { + const t = stack[i] + if (this.finalized[t]) { + continue + } + const node = this.nodesSparseArray[t] + this.cycled.push(node) + onCycle(node) + this.getAdjacentNodeIds(t).forEach((s: number) => this.shouldBeUpdatedMapping[s] = true) + this.finalized[t] = true + } + } + + /** + * Processes a single node: gives it the cycle treatment, skips it if it is current, or runs the operating function. + * Finalizes the node unless the operating function returns DEFERRED. + */ + private processNode(t: number, onCycle: (node: T) => void, operatingFunction: (node: T) => NodeProcessingResult): NodeProcessingResult { + const node = this.nodesSparseArray[t] + const adjacentNodes = this.getAdjacentNodeIds(t) + + // The following line is a potential performance bottleneck. + // Array.includes() is O(n) operation, which makes the whole algorithm O(n^2). + // Idea for improvement: use Set[] instead of number[][] for edgesSparseArray. + if (this.sccNonSingletons[t] || adjacentNodes.includes(t)) { + this.cycled.push(node) + onCycle(node) + adjacentNodes.forEach((s: number) => this.shouldBeUpdatedMapping[s] = true) + this.finalized[t] = true + return NodeProcessingResult.CHANGED + } + + if (!this.shouldBeUpdatedMapping[t]) { + this.sorted.push(node) + this.finalized[t] = true + return NodeProcessingResult.UNCHANGED + } + + const result = operatingFunction(node) + + if (result === NodeProcessingResult.DEFERRED) { + return result + } + + this.sorted.push(node) + this.finalized[t] = true + if (result === NodeProcessingResult.CHANGED) { + adjacentNodes.forEach((s: number) => this.shouldBeUpdatedMapping[s] = true) + } + return result } } diff --git a/hyperformula/src/EvaluationPass.ts b/hyperformula/src/EvaluationPass.ts new file mode 100644 index 0000000000..5455f50c0a --- /dev/null +++ b/hyperformula/src/EvaluationPass.ts @@ -0,0 +1,83 @@ +/** + * @license + * Copyright (c) 2025 Handsoncode. All rights reserved. + */ + +import {SimpleCellAddress} from './Cell' +import {CellVertex, DependencyGraph, Vertex} from './DependencyGraph' +import {FormulaVertex, RuntimeDependency} from './DependencyGraph/FormulaVertex' +import {TopSortPass} from './DependencyGraph/Graph' + +const NO_DEPENDENCIES: RuntimeDependency[] = [] + +/** + * Context of the formula evaluations in one top-sort pass. + * Lets functions that discover references at runtime (e.g., INDIRECT) record them and check whether a cell is current. + */ +export class EvaluationPass { + private currentDependent?: FormulaVertex + private currentRecorded?: RuntimeDependency[] + + constructor( + private readonly dependencyGraph: DependencyGraph, + private readonly topSortPass: TopSortPass, + ) { + } + + /** + * The formula being evaluated, or undefined between evaluations. + */ + public get dependent(): FormulaVertex | undefined { + return this.currentDependent + } + + /** + * Starts recording runtime dependencies for an evaluation of the given formula. + */ + public begin(dependent: FormulaVertex): void { + this.currentDependent = dependent + this.currentRecorded = undefined + } + + /** + * Ends the current evaluation and returns the runtime dependencies it recorded, in reading order, without duplicates. + */ + public end(): RuntimeDependency[] { + const recorded = this.currentRecorded ?? NO_DEPENDENCIES + this.currentDependent = undefined + this.currentRecorded = undefined + return recorded + } + + /** + * Returns true if the vertex may still be recomputed later in this pass, i.e., its value is not current yet. + */ + public isPending(vertex: Vertex): boolean { + return this.topSortPass.isPending(vertex) + } + + /** + * Requests that a pending vertex is computed before the evaluated formula is retried. + * Call it right before throwing {@link EVALUATION_DEFERRED}. + */ + public requestPull(vertex: Vertex): void { + this.topSortPass.requestPull(vertex) + } + + /** + * Records that the evaluated formula reads the cell at the given address, and adds the corresponding edge to the graph. + * + * @returns the vertex of the cell that was read + */ + public recordDependency(address: SimpleCellAddress): CellVertex { + if (this.currentDependent === undefined) { + throw new Error('No formula is being evaluated.') + } + const vertex = this.dependencyGraph.addRuntimeDependency(this.currentDependent, address) + this.currentRecorded = this.currentRecorded ?? [] + if (!this.currentRecorded.some(dependency => dependency.vertex === vertex)) { + this.currentRecorded.push({address, vertex}) + } + return vertex + } +} diff --git a/hyperformula/src/Evaluator.ts b/hyperformula/src/Evaluator.ts index f810bee363..09effcbe56 100644 --- a/hyperformula/src/Evaluator.ts +++ b/hyperformula/src/Evaluator.ts @@ -10,12 +10,17 @@ import {Config} from './Config' import {ContentChanges} from './ContentChanges' import {ArrayFormulaVertex, DependencyGraph, RangeVertex, Vertex} from './DependencyGraph' import {FormulaVertex} from './DependencyGraph/FormulaVertex' +import {TopSortPass} from './DependencyGraph/Graph' +import {NodeProcessingResult} from './DependencyGraph/TopSort' +import {EvaluationPass} from './EvaluationPass' +import {EVALUATION_DEFERRED} from './interpreter/EvaluationDeferred' import {Interpreter} from './interpreter/Interpreter' import {InterpreterState} from './interpreter/InterpreterState' import {EmptyValue, getRawValue, InterpreterValue} from './interpreter/InterpreterValue' import {SimpleRangeValue} from './SimpleRangeValue' import {LazilyTransformingAstService} from './LazilyTransformingAstService' import {ColumnSearchStrategy} from './Lookup/SearchStrategy' +import {Maybe} from './Maybe' import {Ast, RelativeDependency} from './parser' import {Statistics, StatType} from './statistics' @@ -32,12 +37,11 @@ export class Evaluator { } public run(): void { - this.stats.start(StatType.TOP_SORT) - const {sorted, cycled} = this.dependencyGraph.topSortWithScc() - this.stats.end(StatType.TOP_SORT) - this.stats.measure(StatType.EVALUATION, () => { - this.recomputeFormulas(cycled, sorted) + this.evaluateInPasses(this.dependencyGraph.graph.getNodes(), + (vertex: Vertex, pass: EvaluationPass) => this.computeVertexOnInitialRun(vertex, pass), + (vertex: Vertex) => this.setCycleErrorOnInitialRun(vertex), + ) }) } @@ -45,8 +49,8 @@ export class Evaluator { const changes = ContentChanges.empty() this.stats.measure(StatType.EVALUATION, () => { - this.dependencyGraph.graph.getTopSortedWithSccSubgraphFrom(vertices, - (vertex: Vertex) => this.recomputeVertex(vertex, changes), + this.evaluateInPasses(vertices, + (vertex: Vertex, pass: EvaluationPass) => this.recomputeVertex(vertex, changes, pass), (vertex: Vertex) => this.processVertexOnCycle(vertex, changes), ) }) @@ -74,25 +78,97 @@ export class Evaluator { return ret } + /** + * Evaluates the graph starting from the seeds. + * + * A formula that reads a cell which is not current yet (see {@link EVALUATION_DEFERRED}) requests that cell. The sort + * computes the cell, and the pending cells it depends on, right away and then retries the formula, all in the same pass. + * + * A formula on a cycle loses its runtime dependencies and is recomputed once more, in another pass, so that a formula + * that no longer reads a cell on the cycle can leave it. If it reads one again, it keeps #CYCLE!. + */ + private evaluateInPasses( + seeds: Vertex[], + operatingFunction: (vertex: Vertex, pass: EvaluationPass) => NodeProcessingResult, + onCycle: (vertex: Vertex) => void, + ): void { + const retriedAfterCycle = new Set() + let nextSeeds = seeds + + while (nextSeeds.length > 0) { + const retrySeeds: Vertex[] = [] + let evaluationPass: Maybe = undefined + const {deferred} = this.dependencyGraph.graph.getTopSortedWithSccSubgraphFrom(nextSeeds, + (vertex: Vertex, topSortPass: TopSortPass) => { + evaluationPass = evaluationPass ?? new EvaluationPass(this.dependencyGraph, topSortPass) + return operatingFunction(vertex, evaluationPass) + }, + (vertex: Vertex) => { + onCycle(vertex) + if (vertex instanceof FormulaVertex && !retriedAfterCycle.has(vertex) && this.dependencyGraph.clearRuntimeDependencies(vertex)) { + retriedAfterCycle.add(vertex) + retrySeeds.push(vertex) + } + }, + ) + nextSeeds = [...deferred, ...retrySeeds] + } + } + + /** + * Computes the value of a vertex during the initial evaluation of the whole graph. + */ + private computeVertexOnInitialRun(vertex: Vertex, pass: EvaluationPass): NodeProcessingResult { + if (vertex instanceof FormulaVertex) { + const address = vertex.getAddress(this.lazilyTransformingAstService) + const previousValue = vertex.valueOrUndef() + const newCellValue = this.recomputeFormulaVertexValue(vertex, address, pass) + if (newCellValue === undefined) { + return NodeProcessingResult.DEFERRED + } + if (previousValue === undefined) { + this.columnSearch.add(getRawValue(newCellValue), address) + } else { + // recomputed in a retry pass after a cycle + this.columnSearch.change(getRawValue(previousValue), getRawValue(newCellValue), address) + } + } else if (vertex instanceof RangeVertex) { + vertex.clearCache() + } + return NodeProcessingResult.CHANGED + } + + /** + * Sets the cycle error on a vertex that is part of a cycle, during the initial evaluation of the whole graph. + */ + private setCycleErrorOnInitialRun(vertex: Vertex): void { + if (vertex instanceof FormulaVertex) { + vertex.setCellValue(new CellError(ErrorType.CYCLE, undefined, vertex)) + } + } + /** * Recalculates the value of a single vertex assuming its dependencies have already been recalculated */ - private recomputeVertex(vertex: Vertex, changes: ContentChanges): boolean { + private recomputeVertex(vertex: Vertex, changes: ContentChanges, pass: EvaluationPass): NodeProcessingResult { if (vertex instanceof FormulaVertex) { + const address = vertex.getAddress(this.lazilyTransformingAstService) const currentValue = vertex.isComputed() ? vertex.getCellValue() : undefined - const newCellValue = this.recomputeFormulaVertexValue(vertex) + const newCellValue = this.recomputeFormulaVertexValue(vertex, address, pass) + if (newCellValue === undefined) { + return NodeProcessingResult.DEFERRED + } if (newCellValue !== currentValue) { - const address = vertex.getAddress(this.lazilyTransformingAstService) changes.addChange(newCellValue, address) this.columnSearch.change(getRawValue(currentValue), getRawValue(newCellValue), address) - return true + return NodeProcessingResult.CHANGED } - return false + return NodeProcessingResult.UNCHANGED } else if (vertex instanceof RangeVertex) { vertex.clearCache() - return true + return NodeProcessingResult.CHANGED } else { - return true + return NodeProcessingResult.CHANGED } } @@ -112,34 +188,34 @@ export class Evaluator { } /** - * Recalculates formulas in the topological sort order + * Evaluates a formula vertex and stores its value and runtime dependencies. + * + * @returns the new value, or undefined if the evaluation was deferred to a later pass */ - private recomputeFormulas(cycled: Vertex[], sorted: Vertex[]): void { - cycled.forEach((vertex: Vertex) => { - if (vertex instanceof FormulaVertex) { - vertex.setCellValue(new CellError(ErrorType.CYCLE, undefined, vertex)) - } - }) - sorted.forEach((vertex: Vertex) => { - if (vertex instanceof FormulaVertex) { - const newCellValue = this.recomputeFormulaVertexValue(vertex) - const address = vertex.getAddress(this.lazilyTransformingAstService) - this.columnSearch.add(getRawValue(newCellValue), address) - } else if (vertex instanceof RangeVertex) { - vertex.clearCache() - } - }) - } - - private recomputeFormulaVertexValue(vertex: FormulaVertex): InterpreterValue { - const address = vertex.getAddress(this.lazilyTransformingAstService) + private recomputeFormulaVertexValue(vertex: FormulaVertex, address: SimpleCellAddress, evaluationPass: EvaluationPass): Maybe { if (vertex instanceof ArrayFormulaVertex && (vertex.array.size.isRef || !this.dependencyGraph.isThereSpaceForArray(vertex))) { + this.dependencyGraph.clearRuntimeDependencies(vertex) return vertex.setNoSpace() - } else { - const formula = vertex.getFormula(this.lazilyTransformingAstService) - const newCellValue = this.evaluateAstToCellValue(formula, new InterpreterState(address, this.config.useArrayArithmetic, vertex)) - return vertex.setCellValue(newCellValue) } + + const formula = vertex.getFormula(this.lazilyTransformingAstService) + let newCellValue: InterpreterValue + evaluationPass.begin(vertex) + this.interpreter.evaluationPass = evaluationPass + try { + newCellValue = this.evaluateAstToCellValue(formula, new InterpreterState(address, this.config.useArrayArithmetic, vertex)) + } catch (e) { + this.dependencyGraph.keepRuntimeDependencies(vertex, evaluationPass.end()) + if (e === EVALUATION_DEFERRED) { + return undefined + } + throw e + } finally { + this.interpreter.evaluationPass = undefined + } + + this.dependencyGraph.commitRuntimeDependencies(vertex, evaluationPass.end()) + return vertex.setCellValue(newCellValue) } private evaluateAstToCellValue(ast: Ast, state: InterpreterState): InterpreterValue { diff --git a/hyperformula/src/GraphBuilder.ts b/hyperformula/src/GraphBuilder.ts index 6d54d0e54e..9d94204a76 100644 --- a/hyperformula/src/GraphBuilder.ts +++ b/hyperformula/src/GraphBuilder.ts @@ -112,6 +112,12 @@ export class SimpleStrategy implements GraphBuilderStrategy { const vertex = new ArrayFormulaVertex(parseResult.ast, address, new ArraySize(size.width, size.height)) dependencies.set(vertex, absolutizeDependencies(parseResult.dependencies, address)) this.dependencyGraph.addArrayVertex(address, vertex) + if (parseResult.hasVolatileFunction) { + this.dependencyGraph.markAsVolatile(vertex) + } + if (parseResult.hasStructuralChangeFunction) { + this.dependencyGraph.markAsDependentOnStructureChange(vertex) + } } } } else if (parsedCellContent instanceof CellContent.Empty) { diff --git a/hyperformula/src/Operations.ts b/hyperformula/src/Operations.ts index df6d41264f..1fa096c21f 100644 --- a/hyperformula/src/Operations.ts +++ b/hyperformula/src/Operations.ts @@ -284,6 +284,8 @@ export class Operations { mergedPlaceholderSheetId?: number, } { const { previousDisplayName, mergedWithPlaceholderSheet } = this.sheetMapping.renameSheet(sheetId, newName) + // a formula like INDIRECT("OldName!A1") must resolve again + this.dependencyGraph.markStructureDependentFormulasAsDirty() let version: number | undefined if (mergedWithPlaceholderSheet !== undefined) { diff --git a/hyperformula/src/error-message.ts b/hyperformula/src/error-message.ts index d42040901c..aa1f61a9bf 100644 --- a/hyperformula/src/error-message.ts +++ b/hyperformula/src/error-message.ts @@ -38,6 +38,7 @@ export class ErrorMessage { public static CellRefExpected = 'Cell reference expected.' public static EmptyRange = 'Empty range not allowed.' public static BadRef = 'Address is not correct.' + public static R1C1NotSupported = 'R1C1 notation is not supported.' public static NumberRange = 'Number-only range expected.' public static ValueNotFound = 'Value not found.' public static ValueBaseLarge = 'Value in base too large.' diff --git a/hyperformula/src/i18n/languages/enGB.ts b/hyperformula/src/i18n/languages/enGB.ts index d271b732cd..b5ef33e0de 100644 --- a/hyperformula/src/i18n/languages/enGB.ts +++ b/hyperformula/src/i18n/languages/enGB.ts @@ -123,6 +123,7 @@ const dictionary: RawTranslationPackage = { IFNA: 'IFNA', IFS: 'IFS', INDEX: 'INDEX', + INDIRECT: 'INDIRECT', INT: 'INT', INTERVAL: 'INTERVAL', IPMT: 'IPMT', diff --git a/hyperformula/src/interpreter/EvaluationDeferred.ts b/hyperformula/src/interpreter/EvaluationDeferred.ts new file mode 100644 index 0000000000..1cab83402b --- /dev/null +++ b/hyperformula/src/interpreter/EvaluationDeferred.ts @@ -0,0 +1,20 @@ +/** + * @license + * Copyright (c) 2025 Handsoncode. All rights reserved. + */ + +/** + * Signal thrown by a function that needs the value of a cell that is not current yet in the running evaluation pass. + * The evaluator catches it and recomputes the formula in a later pass. Plugins must not catch it. + */ +export class EvaluationDeferred extends Error { + constructor() { + super('Evaluation deferred to a later pass.') + this.name = 'EvaluationDeferred' + } +} + +/** + * The only instance of {@link EvaluationDeferred}. Created once, so throwing it does not capture a stack trace each time. + */ +export const EVALUATION_DEFERRED = new EvaluationDeferred() diff --git a/hyperformula/src/interpreter/Interpreter.ts b/hyperformula/src/interpreter/Interpreter.ts index 6444332537..b4056ff82d 100644 --- a/hyperformula/src/interpreter/Interpreter.ts +++ b/hyperformula/src/interpreter/Interpreter.ts @@ -8,6 +8,7 @@ import {ArraySizePredictor} from '../ArraySize' import {ArrayValue, NotComputedArray} from '../ArrayValue' import {CellError, ErrorType, isColOrRowInvalid} from '../Cell' import {Config} from '../Config' +import {EvaluationPass} from '../EvaluationPass' import {DateTimeHelper} from '../DateTimeHelper' import {DependencyGraph} from '../DependencyGraph' import {FormulaVertex} from '../DependencyGraph/FormulaVertex' @@ -44,6 +45,11 @@ import { AddressWithSheet } from '../parser/Address' export class Interpreter { public readonly criterionBuilder: CriterionBuilder + /** + * Context of the formula evaluation in progress, set by the Evaluator. Undefined outside of evaluation passes + * (e.g., in calculateFormula). + */ + public evaluationPass?: EvaluationPass /** Decides which function calls the license stops; see {@link FunctionCallLicenseGate}. */ private readonly functionCallLicenseGate: FunctionCallLicenseGate diff --git a/hyperformula/src/interpreter/functionMetadata/categories/lookup-and-reference.ts b/hyperformula/src/interpreter/functionMetadata/categories/lookup-and-reference.ts index ef6b848d7f..5dee161eb0 100644 --- a/hyperformula/src/interpreter/functionMetadata/categories/lookup-and-reference.ts +++ b/hyperformula/src/interpreter/functionMetadata/categories/lookup-and-reference.ts @@ -81,6 +81,13 @@ export const LOOKUP_AND_REFERENCE_DOCS: Record = { documentationUrl: 'https://hyperformula.handsontable.com/docs/guide/built-in-functions.html', examples: ['=INDEX(A1:C10, 2, 3)', '=INDEX(A1:A10, 5)'], }, + INDIRECT: { + category: 'Lookup and reference', + shortDescription: 'Returns the value of the cell whose address is given as text. Only single-cell A1 addresses are supported.', + parameters: [{name: 'ref_text', description: 'The address of a single cell as text, in A1 notation, optionally with a sheet name.'}, {name: 'a1', description: 'TRUE (default) for A1 notation. FALSE (R1C1 notation) is not supported and returns #REF!.'}], + documentationUrl: 'https://hyperformula.handsontable.com/docs/guide/built-in-functions.html', + examples: ['=INDIRECT("B1")', '=INDIRECT("Sheet2!$A$1")'], + }, MATCH: { category: 'Lookup and reference', shortDescription: 'Returns the relative position of an item in an array that matches a specified value.', diff --git a/hyperformula/src/interpreter/plugin/IndirectPlugin.ts b/hyperformula/src/interpreter/plugin/IndirectPlugin.ts new file mode 100644 index 0000000000..e535e02e36 --- /dev/null +++ b/hyperformula/src/interpreter/plugin/IndirectPlugin.ts @@ -0,0 +1,97 @@ +/** + * @license + * Copyright (c) 2025 Handsoncode. All rights reserved. + */ + +import {CellError, ErrorType, SimpleCellAddress} from '../../Cell' +import {FormulaVertex} from '../../DependencyGraph/FormulaVertex' +import {ErrorMessage} from '../../error-message' +import {ProcedureAst} from '../../parser' +import {simpleCellAddressFromString} from '../../parser/addressRepresentationConverters' +import {EVALUATION_DEFERRED} from '../EvaluationDeferred' +import {InterpreterState} from '../InterpreterState' +import {InterpreterValue} from '../InterpreterValue' +import {FunctionArgumentType, FunctionPlugin, FunctionPluginTypecheck, ImplementedFunctions} from './FunctionPlugin' + +export class IndirectPlugin extends FunctionPlugin implements FunctionPluginTypecheck { + public static implementedFunctions: ImplementedFunctions = { + 'INDIRECT': { + method: 'indirect', + parameters: [ + {argumentType: FunctionArgumentType.STRING}, + {argumentType: FunctionArgumentType.BOOLEAN, optionalArg: true, defaultValue: true, emptyAsDefault: true}, + ], + // Not volatile: a formula is recomputed through the edges to its selector and to the cell it read. Edits that + // change what an address points to without changing a cell (structure, sheets) mark it dirty instead. + isDependentOnSheetStructureChange: true, + vectorizationForbidden: true, + }, + } + + /** + * Corresponds to INDIRECT(ref_text, [a1]) + * + * Returns the value of the cell whose address is given as text. Supports single-cell A1 addresses, + * optionally with a sheet name. + * + * @param ast + * @param state + */ + public indirect(ast: ProcedureAst, state: InterpreterState): InterpreterValue { + return this.runFunction(ast.args, state, this.metadata('INDIRECT'), (refText: string, a1: boolean): InterpreterValue => { + if (!a1) { + return new CellError(ErrorType.REF, ErrorMessage.R1C1NotSupported) + } + + const address = simpleCellAddressFromString( + (sheetName: string) => this.dependencyGraph.sheetMapping.getSheetId(sheetName), + refText, + state.formulaAddress.sheet, + ) + + if (address === undefined || !this.dependencyGraph.sheetMapping.hasSheetWithId(address.sheet) || !this.isWithinSheetLimits(address)) { + return new CellError(ErrorType.REF, ErrorMessage.BadRef) + } + + return this.readCell(address, state) + }) + } + + /** + * Returns true if the address is within the limits set by the maxRows and maxColumns options. + */ + private isWithinSheetLimits(address: SimpleCellAddress): boolean { + return address.row >= 0 && address.col >= 0 && address.row < this.config.maxRows && address.col < this.config.maxColumns + } + + /** + * Reads the value of a cell, recording it as a runtime dependency of the evaluated formula. + * If the cell is a formula that is not current yet in the running pass, requests it and defers the evaluation: + * the formula is retried once the cell is computed. + */ + private readCell(address: SimpleCellAddress, state: InterpreterState): InterpreterValue { + const pass = this.interpreter.evaluationPass + + if (pass === undefined || state.formulaVertex === undefined || pass.dependent !== state.formulaVertex) { + return this.dependencyGraph.getCellValue(address) + } + + if (this.dependencyGraph.addressMapping.getCell(address) === state.formulaVertex) { + return new CellError(ErrorType.CYCLE, undefined, state.formulaVertex) + } + + const vertex = pass.recordDependency(address) + + if (vertex instanceof FormulaVertex) { + if (pass.isPending(vertex)) { + pass.requestPull(vertex) + throw EVALUATION_DEFERRED + } + if (!vertex.isComputed()) { + return new CellError(ErrorType.REF, ErrorMessage.BadRef) + } + } + + return this.dependencyGraph.getCellValue(address) + } +} diff --git a/hyperformula/src/interpreter/plugin/index.ts b/hyperformula/src/interpreter/plugin/index.ts index 2a3005375b..bab3da9759 100644 --- a/hyperformula/src/interpreter/plugin/index.ts +++ b/hyperformula/src/interpreter/plugin/index.ts @@ -21,6 +21,7 @@ export {ExpPlugin} from './ExpPlugin' export {FinancialPlugin} from './FinancialPlugin' export {FormulaTextPlugin} from './FormulaTextPlugin' export {HyperlinkPlugin} from './HyperlinkPlugin' +export {IndirectPlugin} from './IndirectPlugin' export {InformationPlugin} from './InformationPlugin' export {IsEvenPlugin} from './IsEvenPlugin' export {IsOddPlugin} from './IsOddPlugin' diff --git a/hyperformula/src/license/functionCapabilities.ts b/hyperformula/src/license/functionCapabilities.ts index a26733ec5f..85e0258cdc 100644 --- a/hyperformula/src/license/functionCapabilities.ts +++ b/hyperformula/src/license/functionCapabilities.ts @@ -88,7 +88,7 @@ const UNGROUPED_FUNCTIONS = [ 'GAMMA.INV', 'GAMMALN', 'GAUSS', 'GCD', 'GEOMEAN', 'HARMEAN', 'HEX2BIN', 'HEX2OCT', 'HYPGEOM.DIST', 'IMABS', 'IMAGINARY', 'IMARGUMENT', 'IMCONJUGATE', 'IMCOS', 'IMCOSH', 'IMCOT', 'IMCSC', 'IMCSCH', 'IMDIV', 'IMEXP', 'IMLN', 'IMLOG10', 'IMLOG2', 'IMPOWER', 'IMPRODUCT', 'IMREAL', 'IMSEC', 'IMSECH', 'IMSIN', - 'IMSINH', 'IMSQRT', 'IMSUB', 'IMSUM', 'IMTAN', 'INTERVAL', 'ISBINARY', 'ISFORMULA', 'ISNONTEXT', 'ISPMT', + 'IMSINH', 'IMSQRT', 'IMSUB', 'IMSUM', 'IMTAN', 'INDIRECT', 'INTERVAL', 'ISBINARY', 'ISFORMULA', 'ISNONTEXT', 'ISPMT', 'ISREF', 'LCM', 'LOG10', 'LOGNORM.DIST', 'LOGNORM.INV', 'MAXA', 'MAXPOOL', 'MEDIANPOOL', 'MINA', 'MIRR', 'MMULT', 'MULTINOMIAL', 'NEGBINOM.DIST', 'NETWORKDAYS.INTL', 'NOMINAL', 'NORM.DIST', 'NORM.INV', 'NORM.S.DIST', 'NORM.S.INV', 'NPER', 'OCT2BIN', 'OCT2DEC', 'OCT2HEX', 'PDURATION', 'PERCENTILE.EXC', From 53170c3f5b24590a1280377ba8e1033cdf0740e0 Mon Sep 17 00:00:00 2001 From: tobiadefami Date: Sat, 24 Oct 2026 15:41:34 +0100 Subject: [PATCH 2/3] INDIRECT phase 2: ranges, named expressions and R1C1 references INDIRECT now accepts an A1 range, a named expression and R1C1 text (absolute, relative, ranges, sheet-qualified). A range is read lazily: its cells become a runtime dependency of the formula, through the range vertex, the first time the range's values or address are used, so ROWS and COLUMNS read only its size. Changes to named expressions mark INDIRECT formulas for recalculation. Co-Authored-By: Claude Opus 5.5 --- .../src/DependencyGraph/DependencyGraph.ts | 136 ++++++++----- .../src/DependencyGraph/FormulaVertex.ts | 11 +- hyperformula/src/DependencyGraph/TopSort.ts | 74 +++++--- hyperformula/src/EvaluationPass.ts | 21 +- hyperformula/src/Operations.ts | 6 + hyperformula/src/SimpleRangeValue.ts | 47 ++++- hyperformula/src/error-message.ts | 1 - .../src/interpreter/plugin/IndirectPlugin.ts | 179 ++++++++++++++++-- 8 files changed, 368 insertions(+), 107 deletions(-) diff --git a/hyperformula/src/DependencyGraph/DependencyGraph.ts b/hyperformula/src/DependencyGraph/DependencyGraph.ts index 01299a0ad0..67b113cae4 100644 --- a/hyperformula/src/DependencyGraph/DependencyGraph.ts +++ b/hyperformula/src/DependencyGraph/DependencyGraph.ts @@ -185,52 +185,8 @@ export class DependencyGraph { cellDependencies.forEach((dep: CellDependency) => { if (dep instanceof AbsoluteCellRange) { - const range = dep - - let rangeVertex = this.getRange(range.start, range.end) - if (rangeVertex === undefined) { - rangeVertex = new RangeVertex(range) - this.rangeMapping.addOrUpdateVertex(rangeVertex) - } - - this.graph.addNodeIfNotExists(rangeVertex) - const rangeVertexId = rangeVertex.idInGraph - - if (rangeVertexId === undefined) { - throw new Error('Range vertex not found') - } - - if (!range.isFinite()) { - this.graph.markNodeAsInfiniteRange(rangeVertexId) - } - - const {smallerRangeVertex, restRange} = this.rangeMapping.findSmallerRange(range) - if (smallerRangeVertex !== undefined) { - this.graph.addEdge(smallerRangeVertex, rangeVertexId) - if (rangeVertex.bruteForce) { - rangeVertex.bruteForce = false - for (const cellFromRange of range.addresses(this)) { //if we ever switch heuristic to processing by sorted sizes, this would be unnecessary - this.graph.removeEdge(this.fetchCell(cellFromRange), rangeVertexId) - } - } - } else { - rangeVertex.bruteForce = true - } - - const array = this.arrayMapping.getArray(restRange) - if (array !== undefined) { - this.graph.addEdge(array, rangeVertexId) - } else { - for (const cellFromRange of restRange.addresses(this)) { - const { vertex, id } = this.fetchCellOrCreateEmpty(cellFromRange) - this.graph.addEdge(id ?? vertex, rangeVertexId) - } - } - this.graph.addEdge(rangeVertexId, endVertexId) - - if (range.isFinite()) { - this.correctInfiniteRangesDependenciesByRangeVertex(rangeVertex) - } + const rangeVertex = this.ensureRangeVertex(dep) + this.graph.addEdge(rangeVertex, endVertexId) } else if (dep instanceof NamedExpressionDependency) { const sheetOfVertex = (endVertex as ScalarFormulaVertex).getAddress(this.lazilyTransformingAstService).sheet const { vertex, id } = this.fetchNamedExpressionVertex(dep.name, sheetOfVertex) @@ -242,6 +198,57 @@ export class DependencyGraph { }) } + /** + * Returns the range vertex of a range, creating it if needed, and makes sure it has its edges from its cells + * (or from a smaller range vertex). Safe to call again for a range that already has a vertex. + */ + public ensureRangeVertex(range: AbsoluteCellRange): RangeVertex { + let rangeVertex = this.getRange(range.start, range.end) + if (rangeVertex === undefined) { + rangeVertex = new RangeVertex(range) + this.rangeMapping.addOrUpdateVertex(rangeVertex) + } + + this.graph.addNodeIfNotExists(rangeVertex) + const rangeVertexId = rangeVertex.idInGraph + + if (rangeVertexId === undefined) { + throw new Error('Range vertex not found') + } + + if (!range.isFinite()) { + this.graph.markNodeAsInfiniteRange(rangeVertexId) + } + + const {smallerRangeVertex, restRange} = this.rangeMapping.findSmallerRange(range) + if (smallerRangeVertex !== undefined) { + this.graph.addEdge(smallerRangeVertex, rangeVertexId) + if (rangeVertex.bruteForce) { + rangeVertex.bruteForce = false + for (const cellFromRange of range.addresses(this)) { //if we ever switch heuristic to processing by sorted sizes, this would be unnecessary + this.graph.removeEdge(this.fetchCell(cellFromRange), rangeVertexId) + } + } + } else { + rangeVertex.bruteForce = true + } + + const array = this.arrayMapping.getArray(restRange) + if (array !== undefined) { + this.graph.addEdge(array, rangeVertexId) + } else { + for (const cellFromRange of restRange.addresses(this)) { + const { vertex, id } = this.fetchCellOrCreateEmpty(cellFromRange) + this.graph.addEdge(id ?? vertex, rangeVertexId) + } + } + + if (range.isFinite()) { + this.correctInfiniteRangesDependenciesByRangeVertex(rangeVertex) + } + return rangeVertex + } + public fetchNamedExpressionVertex(expressionName: string, sheetId: number): { vertex: CellVertex, id: Maybe} { const namedExpression = this.namedExpressions.namedExpressionOrPlaceholder(expressionName, sheetId) return this.fetchCellOrCreateEmpty(namedExpression.address) @@ -283,6 +290,21 @@ export class DependencyGraph { return vertex } + /** + * Adds an edge from the range vertex of a range to a formula that read the range during evaluation. + * Creates the range vertex, with its edges from the cells, if the graph does not have it. + * + * @returns the range vertex + */ + public addRuntimeRangeDependency(dependent: FormulaVertex, range: AbsoluteCellRange): RangeVertex { + const rangeVertex = this.ensureRangeVertex(range) + if (!this.graph.existsEdge(rangeVertex, dependent)) { + this.graph.addEdge(rangeVertex, dependent) + this.runtimeEdgesAdded++ + } + return rangeVertex + } + /** * Stores the runtime dependencies recorded in a completed evaluation of a formula. * Removes the edges of stored runtime dependencies that were not read this time, @@ -1448,10 +1470,13 @@ export class DependencyGraph { if (runtimeDependencies === undefined) { return addresses } - const keys = new Set(addresses.map(address => isSimpleCellAddress(address) ? addressKey(address) : undefined)) + const key = (address: SimpleCellAddress | SimpleCellRange) => isSimpleCellAddress(address) + ? addressKey(address) + : `${addressKey(address.start)}:${addressKey(address.end)}` + const keys = new Set(addresses.map(key)) for (const {address, vertex: dependencyVertex} of runtimeDependencies) { - if (this.graph.hasNode(dependencyVertex) && !keys.has(addressKey(address))) { - keys.add(addressKey(address)) + if (this.graph.hasNode(dependencyVertex) && !keys.has(key(address))) { + keys.add(key(address)) addresses.push(address) } } @@ -1470,6 +1495,10 @@ export class DependencyGraph { const [formulaAddress, dependencies] = dependenciesResult for (const dependency of dependencies) { if (dependency instanceof AbsoluteCellRange) { + const rangeVertex = this.rangeMapping.getRangeVertex(dependency.start, dependency.end) + if (rangeVertex !== undefined) { + result.add(rangeVertex) + } continue } const address = dependency instanceof NamedExpressionDependency @@ -1489,7 +1518,14 @@ export class DependencyGraph { private removeRuntimeEdge(dependency: RuntimeDependency, dependent: FormulaVertex): void { const {address, vertex} = dependency this.graph.removeEdgeIfExists(vertex, dependent) - if (vertex instanceof EmptyCellVertex && this.graph.adjacentNodesCount(vertex) === 0 && this.addressMapping.getCell(address) === vertex) { + if (vertex instanceof RangeVertex) { + if (this.graph.adjacentNodesCount(vertex) === 0) { + this.removeVertexAndCleanupDependencies(vertex) + this.rangeMapping.removeVertexIfExists(vertex) + } + return + } + if (vertex instanceof EmptyCellVertex && isSimpleCellAddress(address) && this.graph.adjacentNodesCount(vertex) === 0 && this.addressMapping.getCell(address) === vertex) { this.graph.removeNode(vertex) this.addressMapping.removeCell(address) this.cleanupPlaceholderSheets(new Set([address.sheet])) diff --git a/hyperformula/src/DependencyGraph/FormulaVertex.ts b/hyperformula/src/DependencyGraph/FormulaVertex.ts index 3ad8431743..e5d64c8fcd 100644 --- a/hyperformula/src/DependencyGraph/FormulaVertex.ts +++ b/hyperformula/src/DependencyGraph/FormulaVertex.ts @@ -3,7 +3,7 @@ * Copyright (c) 2025 Handsoncode. All rights reserved. */ -import {AbsoluteCellRange} from '../AbsoluteCellRange' +import {AbsoluteCellRange, SimpleCellRange} from '../AbsoluteCellRange' import {ArraySize} from '../ArraySize' import {ArrayValue, ErroredArray, CellArray, NotComputedArray} from '../ArrayValue' import {CellError, equalSimpleCellAddress, ErrorType, SimpleCellAddress} from '../Cell' @@ -15,16 +15,17 @@ import {Maybe} from '../Maybe' import {Ast} from '../parser' import {ColumnsSpan, RowsSpan} from '../Span' import {CellVertex} from './CellVertex' +import {RangeVertex} from './RangeVertex' /** * A dependency discovered while evaluating a formula (e.g., the target of INDIRECT), as opposed to one collected * from the formula's AST. */ export interface RuntimeDependency { - /** Address read by the formula. Informational: the vertex is the source of truth. */ - address: SimpleCellAddress, - /** Vertex that was read. The graph has an edge from it to the formula vertex. */ - vertex: CellVertex, + /** Address or range read by the formula. Informational: the vertex is the source of truth. */ + address: SimpleCellAddress | SimpleCellRange, + /** Vertex that was read: a cell, or the range vertex of a range. The graph has an edge from it to the formula vertex. */ + vertex: CellVertex | RangeVertex, } /** diff --git a/hyperformula/src/DependencyGraph/TopSort.ts b/hyperformula/src/DependencyGraph/TopSort.ts index cdeb9ec644..b1eb39ae59 100644 --- a/hyperformula/src/DependencyGraph/TopSort.ts +++ b/hyperformula/src/DependencyGraph/TopSort.ts @@ -49,7 +49,9 @@ export class TopSort { private finalized: boolean[] = [] private onPullPath: boolean[] = [] private pendingPrecedents: Map = new Map() - private pullRequest?: number + private pullRequests: number[] = [] + /** Nodes a deferred node requested, still to be computed before it is retried. */ + private requested: Map = new Map() private shouldBeUpdatedMapping: boolean[] = [] private sorted: T[] = [] private cycled: T[] = [] @@ -94,11 +96,12 @@ export class TopSort { } /** - * Requests that a pending node is computed before the node being processed is retried. - * The operating function calls it right before it returns {@link NodeProcessingResult.DEFERRED}. + * Requests that a pending node is computed before the node being processed is retried. Can be called several times + * for one evaluation, e.g., for every pending cell of a range. The operating function calls it right before it + * returns {@link NodeProcessingResult.DEFERRED}. */ public requestPull(id: number): void { - this.pullRequest = id + this.pullRequests.push(id) } /** @@ -219,7 +222,7 @@ export class TopSort { if (this.finalized[t] || this.nodesSparseArray[t] === undefined) { continue } - this.pullRequest = undefined + this.pullRequests = [] if (this.processNode(t, onCycle, operatingFunction) === NodeProcessingResult.DEFERRED) { this.pullFor(t, onCycle, operatingFunction) } @@ -229,12 +232,14 @@ export class TopSort { } /** - * Called after the root node was deferred: pulls the pending nodes it reads and retries it until it is finalized. + * Called after the root node was deferred: computes the pending nodes it requested and retries it until it is + * finalized. Nodes on the stack form a chain: each one waits for the one above it, so reaching a node already on + * the stack is a cycle. */ private pullFor(root: number, onCycle: (node: T) => void, operatingFunction: (node: T) => NodeProcessingResult): void { const stack: number[] = [root] this.onPullPath[root] = true - this.pushRequestedNode(stack, onCycle) + this.recordRequests(root) while (stack.length > 0) { const t = stack[stack.length - 1] @@ -242,44 +247,55 @@ export class TopSort { if (this.finalized[t] || this.nodesSparseArray[t] === undefined) { stack.pop() this.onPullPath[t] = false + this.requested.delete(t) continue } - if (t !== root) { - const precedent = this.nextPendingPrecedent(t) - if (precedent !== undefined) { - if (this.onPullPath[precedent]) { - this.finalizeCycle(stack, precedent, onCycle) - } else { - stack.push(precedent) - this.onPullPath[precedent] = true - } - continue + const next = this.nextRequested(t) ?? (t !== root ? this.nextPendingPrecedent(t) : undefined) + if (next !== undefined) { + if (this.onPullPath[next]) { + this.finalizeCycle(stack, next, onCycle) + } else { + stack.push(next) + this.onPullPath[next] = true } + continue } - this.pullRequest = undefined + this.pullRequests = [] if (this.processNode(t, onCycle, operatingFunction) === NodeProcessingResult.DEFERRED) { - this.pushRequestedNode(stack, onCycle) + this.recordRequests(t) } } } /** - * Pushes the node requested by the deferred node on top of the stack, or gives the cycle treatment to the pull path - * if the requested node is already on it. + * Stores the nodes a just-deferred node requested, to be computed one by one before it is retried. */ - private pushRequestedNode(stack: number[], onCycle: (node: T) => void): void { - const target = this.pullRequest - if (target === undefined || !this.isPending(target)) { + private recordRequests(t: number): void { + const pending = [...new Set(this.pullRequests)].filter(id => this.isPending(id)) + if (pending.length === 0) { throw new Error('A node was deferred without requesting a pending node.') } - if (this.onPullPath[target]) { - this.finalizeCycle(stack, target, onCycle) - } else { - stack.push(target) - this.onPullPath[target] = true + this.requested.set(t, pending.reverse()) + } + + /** + * Returns a node the given node requested that is still pending, or undefined if there is none. + */ + private nextRequested(t: number): number | undefined { + const requested = this.requested.get(t) + if (requested === undefined) { + return undefined + } + while (requested.length > 0 && !this.isPending(requested[requested.length - 1])) { + requested.pop() + } + if (requested.length === 0) { + this.requested.delete(t) + return undefined } + return requested[requested.length - 1] } /** diff --git a/hyperformula/src/EvaluationPass.ts b/hyperformula/src/EvaluationPass.ts index 5455f50c0a..172bb4fea5 100644 --- a/hyperformula/src/EvaluationPass.ts +++ b/hyperformula/src/EvaluationPass.ts @@ -3,8 +3,9 @@ * Copyright (c) 2025 Handsoncode. All rights reserved. */ +import {AbsoluteCellRange, simpleCellRange} from './AbsoluteCellRange' import {SimpleCellAddress} from './Cell' -import {CellVertex, DependencyGraph, Vertex} from './DependencyGraph' +import {CellVertex, DependencyGraph, RangeVertex, Vertex} from './DependencyGraph' import {FormulaVertex, RuntimeDependency} from './DependencyGraph/FormulaVertex' import {TopSortPass} from './DependencyGraph/Graph' @@ -64,6 +65,24 @@ export class EvaluationPass { this.topSortPass.requestPull(vertex) } + /** + * Records that the evaluated formula reads the given range, and adds an edge from the range's vertex to the graph, + * creating the range vertex if needed. + * + * @returns the range vertex + */ + public recordRangeDependency(range: AbsoluteCellRange): RangeVertex { + if (this.currentDependent === undefined) { + throw new Error('No formula is being evaluated.') + } + const vertex = this.dependencyGraph.addRuntimeRangeDependency(this.currentDependent, range) + this.currentRecorded = this.currentRecorded ?? [] + if (!this.currentRecorded.some(dependency => dependency.vertex === vertex)) { + this.currentRecorded.push({address: simpleCellRange(range.start, range.end), vertex}) + } + return vertex + } + /** * Records that the evaluated formula reads the cell at the given address, and adds the corresponding edge to the graph. * diff --git a/hyperformula/src/Operations.ts b/hyperformula/src/Operations.ts index 1fa096c21f..9d883ac352 100644 --- a/hyperformula/src/Operations.ts +++ b/hyperformula/src/Operations.ts @@ -423,6 +423,8 @@ export class Operations { const namedExpression = this.namedExpressions.addNamedExpression(expressionName, sheetId, options) this.storeNamedExpressionInCell(namedExpression.address, expression) this.adjustNamedExpressionEdges(namedExpression, expressionName, sheetId) + // a formula like INDIRECT("Name") may now resolve + this.dependencyGraph.markStructureDependentFormulasAsDirty() } public restoreNamedExpression(namedExpression: InternalNamedExpression, content: ClipboardCell, sheetId?: number) { @@ -430,6 +432,7 @@ export class Operations { this.restoreCell(namedExpression.address, content) const restoredNamedExpression = this.namedExpressions.restoreNamedExpression(namedExpression, sheetId) this.adjustNamedExpressionEdges(restoredNamedExpression, expressionName, sheetId) + this.dependencyGraph.markStructureDependentFormulasAsDirty() } public changeNamedExpressionExpression(expressionName: string, newExpression: RawCellContent, sheetId?: number, options?: NamedExpressionOptions): [InternalNamedExpression, ClipboardCell] { @@ -443,6 +446,8 @@ export class Operations { namedExpression.options = options const content = this.getClipboardCell(namedExpression.address) this.storeNamedExpressionInCell(namedExpression.address, newExpression) + // a formula like INDIRECT("Name") must resolve again + this.dependencyGraph.markStructureDependentFormulasAsDirty() return [oldNamedExpression, content] } @@ -452,6 +457,7 @@ export class Operations { throw new NamedExpressionDoesNotExistError(expressionName) } this.namedExpressions.remove(namedExpression.displayName, sheetId) + this.dependencyGraph.markStructureDependentFormulasAsDirty() const content = this.getClipboardCell(namedExpression.address) if (sheetId !== undefined) { const globalNamedExpression = this.namedExpressions.workbookNamedExpressionOrPlaceholder(expressionName) diff --git a/hyperformula/src/SimpleRangeValue.ts b/hyperformula/src/SimpleRangeValue.ts index e7af429b2b..a1f256f36d 100644 --- a/hyperformula/src/SimpleRangeValue.ts +++ b/hyperformula/src/SimpleRangeValue.ts @@ -20,25 +20,38 @@ export class SimpleRangeValue { */ public readonly size: ArraySize + private readonly _range?: AbsoluteCellRange + + /** + * Runs before the range's cells are first read, e.g., to make sure they are computed. May throw to defer the + * evaluation that reads them; it runs again on the next read. + */ + private beforeRead?: () => void + /** * In most cases, it's more convenient to create a `SimpleRangeValue` object * by calling one of the [static factory methods](#fromrange). */ constructor( private _data?: InternalScalarValue[][], - - /** - * A property that represents the address of the range. - */ - public readonly range?: AbsoluteCellRange, + range?: AbsoluteCellRange, private readonly dependencyGraph?: DependencyGraph, private _hasOnlyNumbers?: boolean, ) { + this._range = range this.size = _data === undefined ? new ArraySize(range!.effectiveWidth(dependencyGraph!), range!.effectiveHeight(dependencyGraph!)) : new ArraySize(_data[0].length, _data.length) } + /** + * A property that represents the address of the range. + */ + public get range(): AbsoluteCellRange | undefined { + this.runBeforeRead() + return this._range + } + /** * Returns the range data as a 2D array. */ @@ -75,6 +88,18 @@ export class SimpleRangeValue { return new SimpleRangeValue(undefined, range, dependencyGraph, undefined) } + /** + * A factory method. Like {@link onlyRange}, but runs the given function before the range's cells or address are + * first used (its size can be read without it). Used for ranges that functions discover at runtime, e.g., INDIRECT. + * + * @internal + */ + public static onlyRangeWithReadHook(range: AbsoluteCellRange, dependencyGraph: DependencyGraph, beforeRead: () => void): SimpleRangeValue { + const value = new SimpleRangeValue(undefined, range, dependencyGraph, undefined) + value.beforeRead = beforeRead + return value + } + /** * A factory method. Returns a `SimpleRangeValue` object that contains a single value. */ @@ -86,7 +111,7 @@ export class SimpleRangeValue { * Returns `true` if and only if the `SimpleRangeValue` has no address set. */ public isAdHoc(): boolean { - return this.range === undefined + return this._range === undefined } /** @@ -208,13 +233,21 @@ export class SimpleRangeValue { /** * Computes the range data if it is not computed yet. */ + private runBeforeRead(): void { + if (this.beforeRead !== undefined) { + this.beforeRead() + this.beforeRead = undefined + } + } + private ensureThatComputed(): void { if (this._data !== undefined) { return } + this.runBeforeRead() this._hasOnlyNumbers = true - this._data = this.range!.addressesArrayMap(this.dependencyGraph!, cellFromRange => { + this._data = this._range!.addressesArrayMap(this.dependencyGraph!, cellFromRange => { const value = this.dependencyGraph!.getCellValue(cellFromRange) if (value instanceof SimpleRangeValue) { this._hasOnlyNumbers = false diff --git a/hyperformula/src/error-message.ts b/hyperformula/src/error-message.ts index aa1f61a9bf..d42040901c 100644 --- a/hyperformula/src/error-message.ts +++ b/hyperformula/src/error-message.ts @@ -38,7 +38,6 @@ export class ErrorMessage { public static CellRefExpected = 'Cell reference expected.' public static EmptyRange = 'Empty range not allowed.' public static BadRef = 'Address is not correct.' - public static R1C1NotSupported = 'R1C1 notation is not supported.' public static NumberRange = 'Number-only range expected.' public static ValueNotFound = 'Value not found.' public static ValueBaseLarge = 'Value in base too large.' diff --git a/hyperformula/src/interpreter/plugin/IndirectPlugin.ts b/hyperformula/src/interpreter/plugin/IndirectPlugin.ts index e535e02e36..fafa13f153 100644 --- a/hyperformula/src/interpreter/plugin/IndirectPlugin.ts +++ b/hyperformula/src/interpreter/plugin/IndirectPlugin.ts @@ -3,16 +3,52 @@ * Copyright (c) 2025 Handsoncode. All rights reserved. */ -import {CellError, ErrorType, SimpleCellAddress} from '../../Cell' +import {AbsoluteCellRange, SimpleCellRange, simpleCellRange} from '../../AbsoluteCellRange' +import {CellError, ErrorType, simpleCellAddress, SimpleCellAddress} from '../../Cell' +import {Vertex} from '../../DependencyGraph' import {FormulaVertex} from '../../DependencyGraph/FormulaVertex' import {ErrorMessage} from '../../error-message' -import {ProcedureAst} from '../../parser' -import {simpleCellAddressFromString} from '../../parser/addressRepresentationConverters' +import {Maybe} from '../../Maybe' +import {AstNodeType, ProcedureAst} from '../../parser' +import {simpleCellAddressFromString, simpleCellRangeFromString} from '../../parser/addressRepresentationConverters' +import {SimpleRangeValue} from '../../SimpleRangeValue' import {EVALUATION_DEFERRED} from '../EvaluationDeferred' import {InterpreterState} from '../InterpreterState' import {InterpreterValue} from '../InterpreterValue' import {FunctionArgumentType, FunctionPlugin, FunctionPluginTypecheck, ImplementedFunctions} from './FunctionPlugin' +const R1C1_CELL = /^R(\[-?\d+\]|\d+)?C(\[-?\d+\]|\d+)?$/i + +/** + * Parses one R1C1 cell: R and C followed by an absolute 1-based number, a relative [offset], or nothing (offset 0). + */ +function r1c1CellAddress(text: string, sheet: number, formulaAddress: SimpleCellAddress): Maybe { + const match = R1C1_CELL.exec(text) + if (match === null) { + return undefined + } + const coordinate = (part: string | undefined, base: number): number => { + if (part === undefined) { + return base + } + if (part.startsWith('[')) { + return base + Number(part.slice(1, -1)) + } + return Number(part) - 1 + } + return simpleCellAddress(sheet, coordinate(match[2], formulaAddress.col), coordinate(match[1], formulaAddress.row)) +} + +/** + * Removes the quotes around a sheet name like 'My Sheet', where '' stands for one quote. + */ +function unquoteSheetName(name: string): string { + if (name.length >= 2 && name.startsWith("'") && name.endsWith("'")) { + return name.slice(1, -1).replace(/''/g, "'") + } + return name +} + export class IndirectPlugin extends FunctionPlugin implements FunctionPluginTypecheck { public static implementedFunctions: ImplementedFunctions = { 'INDIRECT': { @@ -31,29 +67,144 @@ export class IndirectPlugin extends FunctionPlugin implements FunctionPluginType /** * Corresponds to INDIRECT(ref_text, [a1]) * - * Returns the value of the cell whose address is given as text. Supports single-cell A1 addresses, - * optionally with a sheet name. + * Returns the value of the cell, or the range, whose address is given as text. Supports A1 addresses of single + * cells and of ranges, optionally with a sheet name, and names of named expressions that refer to a cell or range. + * With a1 = FALSE, the text is an R1C1 reference to a cell or range, or a name. * * @param ast * @param state */ public indirect(ast: ProcedureAst, state: InterpreterState): InterpreterValue { return this.runFunction(ast.args, state, this.metadata('INDIRECT'), (refText: string, a1: boolean): InterpreterValue => { + const resolveSheet = (sheetName: string) => this.dependencyGraph.sheetMapping.getSheetId(sheetName) + if (!a1) { - return new CellError(ErrorType.REF, ErrorMessage.R1C1NotSupported) + const range = this.r1c1Reference(refText, state.formulaAddress) ?? this.namedReference(refText, state.formulaAddress.sheet) + return this.readReference(range, state) } - const address = simpleCellAddressFromString( - (sheetName: string) => this.dependencyGraph.sheetMapping.getSheetId(sheetName), - refText, - state.formulaAddress.sheet, - ) + const address = simpleCellAddressFromString(resolveSheet, refText, state.formulaAddress.sheet) - if (address === undefined || !this.dependencyGraph.sheetMapping.hasSheetWithId(address.sheet) || !this.isWithinSheetLimits(address)) { - return new CellError(ErrorType.REF, ErrorMessage.BadRef) + if (address !== undefined) { + if (!this.isExistingAddress(address)) { + return new CellError(ErrorType.REF, ErrorMessage.BadRef) + } + return this.readCell(address, state) + } + + const range = simpleCellRangeFromString(resolveSheet, refText, state.formulaAddress.sheet) + ?? this.namedReference(refText, state.formulaAddress.sheet) + return this.readReference(range, state) + }) + } + + /** + * Reads a reference given by its two corners: a cell if they are the same, otherwise a range. + * Returns #REF! if there is no reference or it leaves the existing sheets or the sheet limits. + */ + private readReference(range: Maybe, state: InterpreterState): InterpreterValue { + if (range === undefined || !this.isExistingAddress(range.start) || !this.isExistingAddress(range.end)) { + return new CellError(ErrorType.REF, ErrorMessage.BadRef) + } + if (range.start.row === range.end.row && range.start.col === range.end.col) { + return this.readCell(range.start, state) + } + // corners in any order, like Excel: "B2:A1" is A1:B2 + const normalized = AbsoluteCellRange.fromCoordinates(range.start.sheet, + Math.min(range.start.col, range.end.col), Math.min(range.start.row, range.end.row), + Math.max(range.start.col, range.end.col), Math.max(range.start.row, range.end.row)) + return this.readRange(normalized, state) + } + + /** + * Parses an R1C1 reference to a cell or a range of cells, optionally with a sheet name: R2C3, R[-1]C[2] + * (relative to the formula's cell), RC (the formula's own cell), R1C1:R2C2. Whole rows and columns are not + * supported. Returns undefined if the text is not such a reference or names an unknown sheet. + */ + private r1c1Reference(text: string, formulaAddress: SimpleCellAddress): Maybe { + const sheetSeparator = text.lastIndexOf('!') + let sheet = formulaAddress.sheet + let reference = text + if (sheetSeparator >= 0) { + const sheetId = this.dependencyGraph.sheetMapping.getSheetId(unquoteSheetName(text.slice(0, sheetSeparator))) + if (sheetId === undefined) { + return undefined } + sheet = sheetId + reference = text.slice(sheetSeparator + 1) + } + const corners = reference.split(':') + if (corners.length > 2) { + return undefined + } + const start = r1c1CellAddress(corners[0], sheet, formulaAddress) + const end = corners.length === 2 ? r1c1CellAddress(corners[1], sheet, formulaAddress) : start + if (start === undefined || end === undefined) { + return undefined + } + return simpleCellRange(start, end) + } + + /** + * Returns the cell or range a named expression refers to, as a range (a single cell is a one-cell range), + * or undefined if there is no such name in scope or it is not a reference (e.g., =5). + */ + private namedReference(name: string, sheet: number): Maybe { + const namedExpression = this.dependencyGraph.namedExpressions.nearestNamedExpression(name, sheet) + if (namedExpression === undefined) { + return undefined + } + const vertex = this.dependencyGraph.addressMapping.getCell(namedExpression.address) + if (!(vertex instanceof FormulaVertex)) { + return undefined + } + const ast = vertex.getFormula(this.dependencyGraph.lazilyTransformingAstService) + const baseAddress = vertex.getAddress(this.dependencyGraph.lazilyTransformingAstService) + if (ast.type === AstNodeType.CELL_REFERENCE) { + const address = ast.reference.toSimpleCellAddress(baseAddress) + return new AbsoluteCellRange(address, address) + } + if (ast.type === AstNodeType.CELL_RANGE) { + return AbsoluteCellRange.fromAstOrUndef(ast, baseAddress) + } + return undefined + } - return this.readCell(address, state) + /** + * Returns true if the address is on an existing sheet and within the limits set by the maxRows and maxColumns options. + */ + private isExistingAddress(address: SimpleCellAddress): boolean { + return this.dependencyGraph.sheetMapping.hasSheetWithId(address.sheet) && this.isWithinSheetLimits(address) + } + + /** + * Returns the range as a reference. Its cells are read lazily: only when a function uses the values (SUM) or the + * address (lookups, criteria), not when it only needs the size (ROWS), so the range is recorded as a runtime + * dependency of the evaluated formula only then. Pending cells are requested and the evaluation is deferred. + */ + private readRange(range: AbsoluteCellRange, state: InterpreterState): SimpleRangeValue { + const pass = this.interpreter.evaluationPass + const dependent = state.formulaVertex + return SimpleRangeValue.onlyRangeWithReadHook(range, this.dependencyGraph, () => { + if (pass === undefined || dependent === undefined || pass.dependent !== dependent) { + return + } + const rangeVertex = pass.recordRangeDependency(range) + const pending: Vertex[] = [] + if (pass.isPending(rangeVertex)) { + pending.push(rangeVertex) + } else { + for (const address of range.addresses(this.dependencyGraph)) { + const cell = this.dependencyGraph.addressMapping.getCell(address) + if (cell !== undefined && pass.isPending(cell)) { + pending.push(cell) + } + } + } + if (pending.length > 0) { + pending.forEach(vertex => pass.requestPull(vertex)) + throw EVALUATION_DEFERRED + } }) } From 5368efae98ac842a2d43870ebdb276dd51bdeca1 Mon Sep 17 00:00:00 2001 From: tobiadefami Date: Sat, 24 Oct 2026 15:42:01 +0100 Subject: [PATCH 3/3] INDIRECT: fix moved targets, range results and vertex removal during evaluation - Moving a cell that an INDIRECT formula read marks that formula dirty: the address in its text does not move with the cell, so it reads the address again. - A formula whose result is an INDIRECT range records the range as its dependency while it is evaluated, like a literal range; a cell that later reads the stored range no longer reads cells that are not computed yet. - Range and empty cell vertices that lose their last runtime dependent are removed after the evaluation, not during it, while the sort may still hold them. Co-Authored-By: Claude Opus 5.5 --- .../src/DependencyGraph/DependencyGraph.ts | 39 +++++++++++++++---- hyperformula/src/Evaluator.ts | 3 ++ hyperformula/src/SimpleRangeValue.ts | 7 +++- 3 files changed, 40 insertions(+), 9 deletions(-) diff --git a/hyperformula/src/DependencyGraph/DependencyGraph.ts b/hyperformula/src/DependencyGraph/DependencyGraph.ts index 67b113cae4..739ea07e33 100644 --- a/hyperformula/src/DependencyGraph/DependencyGraph.ts +++ b/hyperformula/src/DependencyGraph/DependencyGraph.ts @@ -58,6 +58,10 @@ export class DependencyGraph { * a formula makes progress. */ public runtimeEdgesAdded = 0 + /** + * Vertices that lost a runtime dependent during evaluation, with the address they were read at. See {@link removeUnusedRuntimeVertices}. + */ + private readonly runtimeVerticesToRemoveIfUnused = new Map() constructor( public readonly addressMapping: AddressMapping, @@ -614,6 +618,9 @@ export class DependencyGraph { emptyVertex = emptyVertex ?? this.fetchCellOrCreateEmpty(sourceAddress).vertex this.graph.addEdge(emptyVertex, adjacentNode) this.graph.removeEdge(sourceVertex, adjacentNode) + } else if (adjacentNode instanceof FormulaVertex && adjacentNode.runtimeDependencies?.some(dependency => dependency.vertex === sourceVertex)) { + // A runtime dependency comes from an address that does not move with the cell, so the formula reads it again + this.graph.markNodeAsDirty(adjacentNode) } } if (emptyVertex) { @@ -1515,21 +1522,37 @@ export class DependencyGraph { /** * Removes the edge of a runtime dependency. Removes the read cell's vertex too if it is an empty cell nothing else depends on. */ + /** + * Removes the edge of a runtime dependency. A range vertex or empty cell vertex left without dependents is removed + * later, by {@link removeUnusedRuntimeVertices}: the edge is removed during evaluation, while the sort may still hold the vertex. + */ private removeRuntimeEdge(dependency: RuntimeDependency, dependent: FormulaVertex): void { const {address, vertex} = dependency this.graph.removeEdgeIfExists(vertex, dependent) - if (vertex instanceof RangeVertex) { - if (this.graph.adjacentNodesCount(vertex) === 0) { + if (vertex instanceof RangeVertex || vertex instanceof EmptyCellVertex) { + this.runtimeVerticesToRemoveIfUnused.set(vertex, address) + } + } + + /** + * Removes the range and empty cell vertices that lost a runtime dependent and have no dependents left. + * Called after evaluation. + */ + public removeUnusedRuntimeVertices(): void { + for (const [vertex, address] of this.runtimeVerticesToRemoveIfUnused) { + if (!this.graph.hasNode(vertex) || this.graph.adjacentNodesCount(vertex) > 0) { + continue + } + if (vertex instanceof RangeVertex) { this.removeVertexAndCleanupDependencies(vertex) this.rangeMapping.removeVertexIfExists(vertex) + } else if (isSimpleCellAddress(address) && this.addressMapping.getCell(address) === vertex) { + this.graph.removeNode(vertex) + this.addressMapping.removeCell(address) + this.cleanupPlaceholderSheets(new Set([address.sheet])) } - return - } - if (vertex instanceof EmptyCellVertex && isSimpleCellAddress(address) && this.graph.adjacentNodesCount(vertex) === 0 && this.addressMapping.getCell(address) === vertex) { - this.graph.removeNode(vertex) - this.addressMapping.removeCell(address) - this.cleanupPlaceholderSheets(new Set([address.sheet])) } + this.runtimeVerticesToRemoveIfUnused.clear() } /** diff --git a/hyperformula/src/Evaluator.ts b/hyperformula/src/Evaluator.ts index 09effcbe56..939451f64e 100644 --- a/hyperformula/src/Evaluator.ts +++ b/hyperformula/src/Evaluator.ts @@ -113,6 +113,7 @@ export class Evaluator { ) nextSeeds = [...deferred, ...retrySeeds] } + this.dependencyGraph.removeUnusedRuntimeVertices() } /** @@ -221,6 +222,8 @@ export class Evaluator { private evaluateAstToCellValue(ast: Ast, state: InterpreterState): InterpreterValue { const interpreterValue = this.interpreter.evaluateAst(ast, state) if (interpreterValue instanceof SimpleRangeValue) { + // Formulas that read this cell later read the range's cells, so they become dependencies of this formula now + interpreterValue.runBeforeRead() return interpreterValue } else if (interpreterValue === EmptyValue && this.config.evaluateNullToZero) { return 0 diff --git a/hyperformula/src/SimpleRangeValue.ts b/hyperformula/src/SimpleRangeValue.ts index a1f256f36d..ea515ba590 100644 --- a/hyperformula/src/SimpleRangeValue.ts +++ b/hyperformula/src/SimpleRangeValue.ts @@ -233,7 +233,12 @@ export class SimpleRangeValue { /** * Computes the range data if it is not computed yet. */ - private runBeforeRead(): void { + /** + * Runs the function given to {@link onlyRangeWithReadHook} unless it has already run. + * + * @internal + */ + public runBeforeRead(): void { if (this.beforeRead !== undefined) { this.beforeRead() this.beforeRead = undefined