diff --git a/hyperformula/src/DependencyGraph/DependencyGraph.ts b/hyperformula/src/DependencyGraph/DependencyGraph.ts index 962dedff1a..739ea07e33 100644 --- a/hyperformula/src/DependencyGraph/DependencyGraph.ts +++ b/hyperformula/src/DependencyGraph/DependencyGraph.ts @@ -6,7 +6,7 @@ import {AbsoluteCellRange, SimpleCellRange, simpleCellRange} from '../AbsoluteCellRange' import {absolutizeDependencies} from '../absolutizeDependencies' import {ArraySize} from '../ArraySize' -import {CellError, ErrorType, isSimpleCellAddress, simpleCellAddress, SimpleCellAddress} from '../Cell' +import {addressKey, CellError, ErrorType, isSimpleCellAddress, simpleCellAddress, SimpleCellAddress} from '../Cell' import {RawCellContent} from '../CellContentParser' import {CellDependency} from '../CellDependency' import {Config} from '../Config' @@ -40,7 +40,7 @@ import { import {AddressMapping} from './AddressMapping/AddressMapping' import {ArrayMapping} from './ArrayMapping' import {collectAddressesDependentToRange} from './collectAddressesDependentToRange' -import {FormulaVertex} from './FormulaVertex' +import {FormulaVertex, RuntimeDependency} from './FormulaVertex' import {DependencyQuery, Graph} from './Graph' import {RangeMapping} from './RangeMapping' import {SheetMapping} from './SheetMapping' @@ -53,6 +53,15 @@ export class DependencyGraph { public readonly graph: Graph private changes: ContentChanges = ContentChanges.empty() public readonly sheetReferenceRegistrar: SheetReferenceRegistrar + /** + * Number of runtime dependency edges added to the graph so far. Lets the evaluator check that each pass that defers + * a formula makes progress. + */ + public runtimeEdgesAdded = 0 + /** + * Vertices that lost a runtime dependent during evaluation, with the address they were read at. See {@link removeUnusedRuntimeVertices}. + */ + private readonly runtimeVerticesToRemoveIfUnused = new Map() constructor( public readonly addressMapping: AddressMapping, @@ -180,52 +189,8 @@ export class DependencyGraph { cellDependencies.forEach((dep: CellDependency) => { if (dep instanceof AbsoluteCellRange) { - const range = dep - - let rangeVertex = this.getRange(range.start, range.end) - if (rangeVertex === undefined) { - rangeVertex = new RangeVertex(range) - this.rangeMapping.addOrUpdateVertex(rangeVertex) - } - - this.graph.addNodeIfNotExists(rangeVertex) - const rangeVertexId = rangeVertex.idInGraph - - if (rangeVertexId === undefined) { - throw new Error('Range vertex not found') - } - - if (!range.isFinite()) { - this.graph.markNodeAsInfiniteRange(rangeVertexId) - } - - const {smallerRangeVertex, restRange} = this.rangeMapping.findSmallerRange(range) - if (smallerRangeVertex !== undefined) { - this.graph.addEdge(smallerRangeVertex, rangeVertexId) - if (rangeVertex.bruteForce) { - rangeVertex.bruteForce = false - for (const cellFromRange of range.addresses(this)) { //if we ever switch heuristic to processing by sorted sizes, this would be unnecessary - this.graph.removeEdge(this.fetchCell(cellFromRange), rangeVertexId) - } - } - } else { - rangeVertex.bruteForce = true - } - - const array = this.arrayMapping.getArray(restRange) - if (array !== undefined) { - this.graph.addEdge(array, rangeVertexId) - } else { - for (const cellFromRange of restRange.addresses(this)) { - const { vertex, id } = this.fetchCellOrCreateEmpty(cellFromRange) - this.graph.addEdge(id ?? vertex, rangeVertexId) - } - } - this.graph.addEdge(rangeVertexId, endVertexId) - - if (range.isFinite()) { - this.correctInfiniteRangesDependenciesByRangeVertex(rangeVertex) - } + const rangeVertex = this.ensureRangeVertex(dep) + this.graph.addEdge(rangeVertex, endVertexId) } else if (dep instanceof NamedExpressionDependency) { const sheetOfVertex = (endVertex as ScalarFormulaVertex).getAddress(this.lazilyTransformingAstService).sheet const { vertex, id } = this.fetchNamedExpressionVertex(dep.name, sheetOfVertex) @@ -237,6 +202,57 @@ export class DependencyGraph { }) } + /** + * Returns the range vertex of a range, creating it if needed, and makes sure it has its edges from its cells + * (or from a smaller range vertex). Safe to call again for a range that already has a vertex. + */ + public ensureRangeVertex(range: AbsoluteCellRange): RangeVertex { + let rangeVertex = this.getRange(range.start, range.end) + if (rangeVertex === undefined) { + rangeVertex = new RangeVertex(range) + this.rangeMapping.addOrUpdateVertex(rangeVertex) + } + + this.graph.addNodeIfNotExists(rangeVertex) + const rangeVertexId = rangeVertex.idInGraph + + if (rangeVertexId === undefined) { + throw new Error('Range vertex not found') + } + + if (!range.isFinite()) { + this.graph.markNodeAsInfiniteRange(rangeVertexId) + } + + const {smallerRangeVertex, restRange} = this.rangeMapping.findSmallerRange(range) + if (smallerRangeVertex !== undefined) { + this.graph.addEdge(smallerRangeVertex, rangeVertexId) + if (rangeVertex.bruteForce) { + rangeVertex.bruteForce = false + for (const cellFromRange of range.addresses(this)) { //if we ever switch heuristic to processing by sorted sizes, this would be unnecessary + this.graph.removeEdge(this.fetchCell(cellFromRange), rangeVertexId) + } + } + } else { + rangeVertex.bruteForce = true + } + + const array = this.arrayMapping.getArray(restRange) + if (array !== undefined) { + this.graph.addEdge(array, rangeVertexId) + } else { + for (const cellFromRange of restRange.addresses(this)) { + const { vertex, id } = this.fetchCellOrCreateEmpty(cellFromRange) + this.graph.addEdge(id ?? vertex, rangeVertexId) + } + } + + if (range.isFinite()) { + this.correctInfiniteRangesDependenciesByRangeVertex(rangeVertex) + } + return rangeVertex + } + public fetchNamedExpressionVertex(expressionName: string, sheetId: number): { vertex: CellVertex, id: Maybe} { const namedExpression = this.namedExpressions.namedExpressionOrPlaceholder(expressionName, sheetId) return this.fetchCellOrCreateEmpty(namedExpression.address) @@ -263,6 +279,101 @@ export class DependencyGraph { return { vertex: newVertex, id: newVertexId } } + /** + * Adds an edge from the cell at the given address to a formula that read it during evaluation. + * Creates an empty cell vertex if there is none at the address. + * + * @returns the vertex of the cell that was read + */ + public addRuntimeDependency(dependent: FormulaVertex, address: SimpleCellAddress): CellVertex { + const { vertex, id } = this.fetchCellOrCreateEmpty(address) + if (!this.graph.existsEdge(vertex, dependent)) { + this.graph.addEdge(id ?? vertex, dependent) + this.runtimeEdgesAdded++ + } + return vertex + } + + /** + * Adds an edge from the range vertex of a range to a formula that read the range during evaluation. + * Creates the range vertex, with its edges from the cells, if the graph does not have it. + * + * @returns the range vertex + */ + public addRuntimeRangeDependency(dependent: FormulaVertex, range: AbsoluteCellRange): RangeVertex { + const rangeVertex = this.ensureRangeVertex(range) + if (!this.graph.existsEdge(rangeVertex, dependent)) { + this.graph.addEdge(rangeVertex, dependent) + this.runtimeEdgesAdded++ + } + return rangeVertex + } + + /** + * Stores the runtime dependencies recorded in a completed evaluation of a formula. + * Removes the edges of stored runtime dependencies that were not read this time, + * unless the formula also depends on that cell through its AST. + */ + public commitRuntimeDependencies(dependent: FormulaVertex, recorded: RuntimeDependency[]): void { + if (recorded.length === 0 && dependent.runtimeDependencies === undefined) { + return + } + const stored = dependent.runtimeDependencies ?? [] + const recordedVertices = new Set(recorded.map(dependency => dependency.vertex)) + const stale = stored.filter(dependency => this.graph.hasNode(dependency.vertex) && !recordedVertices.has(dependency.vertex)) + + if (stale.length > 0) { + const staticVertices = this.staticDependencyVertices(dependent) + stale.forEach((dependency) => { + if (!staticVertices.has(dependency.vertex)) { + this.removeRuntimeEdge(dependency, dependent) + } + }) + } + + dependent.runtimeDependencies = recorded.length > 0 ? recorded : undefined + } + + /** + * Adds the runtime dependencies recorded in an interrupted evaluation of a formula to the stored ones, removing none. + */ + public keepRuntimeDependencies(dependent: FormulaVertex, recorded: RuntimeDependency[]): void { + if (recorded.length === 0) { + return + } + const stored = (dependent.runtimeDependencies ?? []).filter(dependency => this.graph.hasNode(dependency.vertex)) + const storedVertices = new Set(stored.map(dependency => dependency.vertex)) + dependent.runtimeDependencies = [...stored, ...recorded.filter(dependency => !storedVertices.has(dependency.vertex))] + } + + /** + * Removes all runtime dependencies of a formula and their edges, except edges the formula also has through its AST. + * + * @returns true if any edge was removed + */ + public clearRuntimeDependencies(dependent: FormulaVertex): boolean { + const stored = dependent.runtimeDependencies + dependent.runtimeDependencies = undefined + if (stored === undefined) { + return false + } + + const live = stored.filter(dependency => this.graph.hasNode(dependency.vertex)) + if (live.length === 0) { + return false + } + + const staticVertices = this.staticDependencyVertices(dependent) + let removedAny = false + live.forEach((dependency) => { + if (!staticVertices.has(dependency.vertex)) { + this.removeRuntimeEdge(dependency, dependent) + removedAny = true + } + }) + return removedAny + } + public removeRows(removedRows: RowsSpan): EagerChangesGraphChangeResult { this.stats.measure(StatType.ADJUSTING_GRAPH, () => { for (const [address, vertex] of this.addressMapping.entriesFromRowsSpan(removedRows)) { @@ -316,6 +427,16 @@ export class DependencyGraph { this.stats.measure(StatType.ADJUSTING_RANGES, () => { this.markAllRangesAsDirtyInSheet(sheetId) }) + + // a formula like INDIRECT("NewSheet!A1") may now resolve + this.addStructuralNodesToChangeSet() + } + + /** + * Marks the formulas that depend on the sheet structure as dirty, e.g., after a sheet is renamed. + */ + public markStructureDependentFormulasAsDirty(): void { + this.addStructuralNodesToChangeSet() } /** @@ -497,6 +618,9 @@ export class DependencyGraph { emptyVertex = emptyVertex ?? this.fetchCellOrCreateEmpty(sourceAddress).vertex this.graph.addEdge(emptyVertex, adjacentNode) this.graph.removeEdge(sourceVertex, adjacentNode) + } else if (adjacentNode instanceof FormulaVertex && adjacentNode.runtimeDependencies?.some(dependency => dependency.vertex === sourceVertex)) { + // A runtime dependency comes from an address that does not move with the cell, so the formula reads it again + this.graph.markNodeAsDirty(adjacentNode) } } if (emptyVertex) { @@ -512,6 +636,7 @@ export class DependencyGraph { for (const adjacentNode of this.graph.adjacentNodes(targetVertex)) { sourceVertex = sourceVertex ?? this.fetchCellOrCreateEmpty(targetAddress).vertex this.graph.addEdge(sourceVertex, adjacentNode) + this.replaceRuntimeDependencyVertex(adjacentNode, targetVertex, sourceVertex) this.graph.markNodeAsDirty(sourceVertex) } this.removeVertex(targetVertex) @@ -695,7 +820,7 @@ export class DependencyGraph { const dependenciesResult = this.formulaDependencyQuery(vertex) if (dependenciesResult !== undefined) { const [address, dependencies] = dependenciesResult - return dependencies.map((dependency: CellDependency) => { + const addresses: (SimpleCellAddress | SimpleCellRange)[] = dependencies.map((dependency: CellDependency) => { if (dependency instanceof NamedExpressionDependency) { return this.namedExpressions.namedExpressionOrPlaceholder(dependency.name, address.sheet).address } else if (isSimpleCellAddress(dependency)) { @@ -704,6 +829,7 @@ export class DependencyGraph { return simpleCellRange(dependency.start, dependency.end) } }) + return this.appendRuntimeDependencyAddresses(vertex as FormulaVertex, addresses) } else { return [] } @@ -867,6 +993,7 @@ export class DependencyGraph { adjNodesStored.forEach((adjacentNode) => { if (this.graph.hasNode(adjacentNode)) { this.graph.addEdge(newNode, adjacentNode) + this.replaceRuntimeDependencyVertex(adjacentNode, oldNode, newNode) } }) } @@ -910,7 +1037,7 @@ export class DependencyGraph { const dependenciesResult = this.formulaDependencyQuery(vertex) if (dependenciesResult !== undefined) { const [address, dependencies] = dependenciesResult - return dependencies.map((dependency: CellDependency) => { + const result: [SimpleCellAddress | SimpleCellRange, Vertex][] = dependencies.map((dependency: CellDependency) => { if (dependency instanceof AbsoluteCellRange) { return [dependency.start, this.rangeMapping.getVertexOrThrow(dependency.start, dependency.end)] } else if (dependency instanceof NamedExpressionDependency) { @@ -920,6 +1047,12 @@ export class DependencyGraph { return [dependency, this.addressMapping.getCellOrThrow(dependency)] } }) + for (const runtimeDependency of (vertex as FormulaVertex).runtimeDependencies ?? []) { + if (this.graph.hasNode(runtimeDependency.vertex)) { + result.push([runtimeDependency.address, runtimeDependency.vertex]) + } + } + return result } else { return [] } @@ -1280,6 +1413,7 @@ export class DependencyGraph { dependents.forEach((adjacentNode) => { if (this.graph.hasNode(adjacentNode)) { this.graph.addEdge(target, adjacentNode) + this.replaceRuntimeDependencyVertex(adjacentNode, source, target) } }) @@ -1315,6 +1449,10 @@ export class DependencyGraph { dependencies.delete(dependency) const [address, vertex] = dependency if (this.graph.hasNode(vertex) && this.graph.adjacentNodesCount(vertex) === 0) { + if (vertex instanceof EmptyCellVertex && isSimpleCellAddress(address) && this.addressMapping.getCell(address) !== vertex) { + // a runtime dependency whose address no longer maps to this vertex + continue + } if (vertex instanceof RangeVertex || vertex instanceof EmptyCellVertex) { this.graph.removeNode(vertex).forEach((candidate) => dependencies.add(candidate)) } @@ -1331,6 +1469,101 @@ export class DependencyGraph { this.cleanupPlaceholderSheets(affectedSheets) } + /** + * Appends the addresses of a formula's runtime dependencies to the addresses collected from its AST, skipping duplicates. + */ + private appendRuntimeDependencyAddresses(vertex: FormulaVertex, addresses: (SimpleCellAddress | SimpleCellRange)[]): (SimpleCellAddress | SimpleCellRange)[] { + const runtimeDependencies = vertex.runtimeDependencies + if (runtimeDependencies === undefined) { + return addresses + } + const key = (address: SimpleCellAddress | SimpleCellRange) => isSimpleCellAddress(address) + ? addressKey(address) + : `${addressKey(address.start)}:${addressKey(address.end)}` + const keys = new Set(addresses.map(key)) + for (const {address, vertex: dependencyVertex} of runtimeDependencies) { + if (this.graph.hasNode(dependencyVertex) && !keys.has(key(address))) { + keys.add(key(address)) + addresses.push(address) + } + } + return addresses + } + + /** + * Returns the cell vertices a formula depends on directly through its AST (cell references and named expressions). + */ + private staticDependencyVertices(vertex: FormulaVertex): Set { + const result = new Set() + const dependenciesResult = this.formulaDependencyQuery(vertex) + if (dependenciesResult === undefined) { + return result + } + const [formulaAddress, dependencies] = dependenciesResult + for (const dependency of dependencies) { + if (dependency instanceof AbsoluteCellRange) { + const rangeVertex = this.rangeMapping.getRangeVertex(dependency.start, dependency.end) + if (rangeVertex !== undefined) { + result.add(rangeVertex) + } + continue + } + const address = dependency instanceof NamedExpressionDependency + ? this.namedExpressions.namedExpressionOrPlaceholder(dependency.name, formulaAddress.sheet).address + : dependency + const cell = this.addressMapping.getCell(address) + if (cell !== undefined) { + result.add(cell) + } + } + return result + } + + /** + * Removes the edge of a runtime dependency. Removes the read cell's vertex too if it is an empty cell nothing else depends on. + */ + /** + * Removes the edge of a runtime dependency. A range vertex or empty cell vertex left without dependents is removed + * later, by {@link removeUnusedRuntimeVertices}: the edge is removed during evaluation, while the sort may still hold the vertex. + */ + private removeRuntimeEdge(dependency: RuntimeDependency, dependent: FormulaVertex): void { + const {address, vertex} = dependency + this.graph.removeEdgeIfExists(vertex, dependent) + if (vertex instanceof RangeVertex || vertex instanceof EmptyCellVertex) { + this.runtimeVerticesToRemoveIfUnused.set(vertex, address) + } + } + + /** + * Removes the range and empty cell vertices that lost a runtime dependent and have no dependents left. + * Called after evaluation. + */ + public removeUnusedRuntimeVertices(): void { + for (const [vertex, address] of this.runtimeVerticesToRemoveIfUnused) { + if (!this.graph.hasNode(vertex) || this.graph.adjacentNodesCount(vertex) > 0) { + continue + } + if (vertex instanceof RangeVertex) { + this.removeVertexAndCleanupDependencies(vertex) + this.rangeMapping.removeVertexIfExists(vertex) + } else if (isSimpleCellAddress(address) && this.addressMapping.getCell(address) === vertex) { + this.graph.removeNode(vertex) + this.addressMapping.removeCell(address) + this.cleanupPlaceholderSheets(new Set([address.sheet])) + } + } + this.runtimeVerticesToRemoveIfUnused.clear() + } + + /** + * Points a formula's runtime dependencies on oldVertex to newVertex. + */ + private replaceRuntimeDependencyVertex(dependent: Vertex, oldVertex: Vertex, newVertex: Vertex): void { + if (dependent instanceof FormulaVertex && oldVertex instanceof CellVertex && newVertex instanceof CellVertex) { + dependent.replaceRuntimeDependencyVertex(oldVertex, newVertex) + } + } + /** * Removes placeholder sheets that have no remaining vertices. */ diff --git a/hyperformula/src/DependencyGraph/FormulaVertex.ts b/hyperformula/src/DependencyGraph/FormulaVertex.ts index 5aa448ffd8..e5d64c8fcd 100644 --- a/hyperformula/src/DependencyGraph/FormulaVertex.ts +++ b/hyperformula/src/DependencyGraph/FormulaVertex.ts @@ -3,7 +3,7 @@ * Copyright (c) 2025 Handsoncode. All rights reserved. */ -import {AbsoluteCellRange} from '../AbsoluteCellRange' +import {AbsoluteCellRange, SimpleCellRange} from '../AbsoluteCellRange' import {ArraySize} from '../ArraySize' import {ArrayValue, ErroredArray, CellArray, NotComputedArray} from '../ArrayValue' import {CellError, equalSimpleCellAddress, ErrorType, SimpleCellAddress} from '../Cell' @@ -15,7 +15,18 @@ import {Maybe} from '../Maybe' import {Ast} from '../parser' import {ColumnsSpan, RowsSpan} from '../Span' import {CellVertex} from './CellVertex' +import {RangeVertex} from './RangeVertex' +/** + * A dependency discovered while evaluating a formula (e.g., the target of INDIRECT), as opposed to one collected + * from the formula's AST. + */ +export interface RuntimeDependency { + /** Address or range read by the formula. Informational: the vertex is the source of truth. */ + address: SimpleCellAddress | SimpleCellRange, + /** Vertex that was read: a cell, or the range vertex of a range. The graph has an edge from it to the formula vertex. */ + vertex: CellVertex | RangeVertex, +} /** * Abstract base class for vertices that contain formulas in the dependency graph. @@ -25,6 +36,11 @@ import {CellVertex} from './CellVertex' * and {@link ArrayFormulaVertex} for array formulas that span multiple cells. */ export abstract class FormulaVertex extends CellVertex { + /** + * Dependencies recorded during the last evaluation of the formula. Undefined for formulas that have none. + */ + public runtimeDependencies?: RuntimeDependency[] + protected constructor( protected formula: Ast, protected cellAddress: SimpleCellAddress, @@ -87,6 +103,20 @@ export abstract class FormulaVertex extends CellVertex { public abstract valueOrUndef(): Maybe public abstract isComputed(): boolean + + /** + * Points runtime dependencies on oldVertex to newVertex, when a cell vertex is replaced by another one. + */ + public replaceRuntimeDependencyVertex(oldVertex: CellVertex, newVertex: CellVertex): void { + if (this.runtimeDependencies === undefined) { + return + } + for (const dependency of this.runtimeDependencies) { + if (dependency.vertex === oldVertex) { + dependency.vertex = newVertex + } + } + } } /** diff --git a/hyperformula/src/DependencyGraph/Graph.ts b/hyperformula/src/DependencyGraph/Graph.ts index 0a4086dcff..7dc8add37b 100644 --- a/hyperformula/src/DependencyGraph/Graph.ts +++ b/hyperformula/src/DependencyGraph/Graph.ts @@ -5,7 +5,7 @@ import { SimpleCellAddress } from '../Cell' import { SimpleCellRange } from '../AbsoluteCellRange' -import { TopSort, TopSortResult } from './TopSort' +import { NodeProcessingResult, TopSort, TopSortResult } from './TopSort' import { ProcessableValue } from './ProcessableValue' export type NodeId = number @@ -15,6 +15,21 @@ export type DependencyQuery = (vertex: Node) => [(SimpleCellAddress | Simp * Interface for nodes that can be stored in the Graph. * The idInGraph property is managed by the Graph class and should not be set externally. */ +/** + * View of a running top-sort pass, passed to the operating function together with the node. + */ +export interface TopSortPass { + /** + * Returns true if the node was reached by the pass and its value may still change later in it. + */ + isPending(node: Node): boolean, + /** + * Requests that a pending node is computed before the node being processed is retried. + * Call it right before returning {@link NodeProcessingResult.DEFERRED}. + */ + requestPull(node: Node): void, +} + export interface GraphNode { idInGraph?: NodeId, } @@ -258,24 +273,41 @@ export class Graph { * Sorts the whole graph topologically. Nodes that are on cycles are kept separate. */ public topSortWithScc(): TopSortResult { - return this.getTopSortedWithSccSubgraphFrom(this.getNodes(), () => true, () => { }) + return this.getTopSortedWithSccSubgraphFrom(this.getNodes(), () => NodeProcessingResult.CHANGED, () => { }) } /** * Sorts the graph topologically. Nodes that are on cycles are kept separate. * * @param {Node[]} modifiedNodes - seed for computation. The algorithm assumes that only these nodes have changed since the last run. - * @param {(node: Node) => boolean} operatingFunction - recomputes value of a node, and returns whether a change occurred + * @param {(node: Node, pass: TopSortPass) => NodeProcessingResult} operatingFunction - recomputes value of a node, and returns the outcome * @param {(node: Node) => void} onCycle - action to be performed when node is on cycle */ public getTopSortedWithSccSubgraphFrom( modifiedNodes: Node[], - operatingFunction: (node: Node) => boolean, + operatingFunction: (node: Node, pass: TopSortPass) => NodeProcessingResult, onCycle: (node: Node) => void ): TopSortResult { - const topSortAlgorithm = new TopSort(this.nodesSparseArray, this.edgesSparseArray) + const precedentIds = (id: NodeId): NodeId[] => { + const node = this.nodesSparseArray[id] + if (node === undefined) { + return [] + } + return this.dependencyQuery(node) + .map(([, dependency]) => dependency.idInGraph) + .filter((dependencyId): dependencyId is NodeId => dependencyId !== undefined) + } + const topSortAlgorithm = new TopSort(this.nodesSparseArray, this.edgesSparseArray, precedentIds) + const pass: TopSortPass = { + isPending: (node: Node) => node.idInGraph !== undefined && topSortAlgorithm.isPending(node.idInGraph), + requestPull: (node: Node) => { + if (node.idInGraph !== undefined) { + topSortAlgorithm.requestPull(node.idInGraph) + } + }, + } const modifiedNodesIds = modifiedNodes.map(node => node.idInGraph).filter(id => id !== undefined) as NodeId[] - return topSortAlgorithm.getTopSortedWithSccSubgraphFrom(modifiedNodesIds, operatingFunction, onCycle) + return topSortAlgorithm.getTopSortedWithSccSubgraphFrom(modifiedNodesIds, (node: Node) => operatingFunction(node, pass), onCycle) } /** diff --git a/hyperformula/src/DependencyGraph/TopSort.ts b/hyperformula/src/DependencyGraph/TopSort.ts index b79e450fce..b1eb39ae59 100644 --- a/hyperformula/src/DependencyGraph/TopSort.ts +++ b/hyperformula/src/DependencyGraph/TopSort.ts @@ -6,6 +6,26 @@ export interface TopSortResult { sorted: T[], cycled: T[], + /** + * Nodes that still have to be recomputed in another pass. Always empty: a node that reads a pending node is + * retried in the same pass, after the pending node is computed. + */ + deferred: T[], +} + +/** + * Outcome of processing a single node during a top-sort pass. + */ +export enum NodeProcessingResult { + /** The node was recomputed and its value changed. */ + CHANGED = 'CHANGED', + /** The node was recomputed and its value did not change. */ + UNCHANGED = 'UNCHANGED', + /** + * The node read a value that is not current yet, and requested that node with {@link TopSort.requestPull}. + * It is recomputed again once the requested node is computed. + */ + DEFERRED = 'DEFERRED', } // node status life cycle: undefined -> ON_STACK -> PROCESSED -> POPPED @@ -26,11 +46,26 @@ export class TopSort { private nodeStatus: NodeVisitStatus[] = [] private order: number[] = [] private sccNonSingletons: boolean[] = [] + private finalized: boolean[] = [] + private onPullPath: boolean[] = [] + private pendingPrecedents: Map = new Map() + private pullRequests: number[] = [] + /** Nodes a deferred node requested, still to be computed before it is retried. */ + private requested: Map = new Map() + private shouldBeUpdatedMapping: boolean[] = [] + private sorted: T[] = [] + private cycled: T[] = [] private timeCounter: number = 0 + /** + * @param nodesSparseArray - nodes by id + * @param edgesSparseArray - ids of the dependents of each node, by id + * @param precedentIds - ids of the nodes a node reads; used only for nodes pulled out of order + */ constructor( private nodesSparseArray: T[] = [], private edgesSparseArray: number[][] = [], + private precedentIds: (id: number) => number[] = () => [], ) { } @@ -39,12 +74,12 @@ export class TopSort { * Returns vertices in order of topological sort, but vertices that are on cycles are kept separate. * * @param modifiedNodes - seed for computation. During engine init run, all of the vertices of grap. In recomputation run, changed vertices. - * @param operatingFunction - recomputes value of a node, and returns whether a change occurred + * @param operatingFunction - recomputes value of a node, and returns the outcome * @param onCycle - action to be performed when node is on cycle */ public getTopSortedWithSccSubgraphFrom( modifiedNodeIds: number[], - operatingFunction: (node: T) => boolean, + operatingFunction: (node: T) => NodeProcessingResult, onCycle: (node: T) => void ): TopSortResult { const modifiedNodeIdsReversed = modifiedNodeIds.reverse() @@ -52,6 +87,23 @@ export class TopSort { return this.postprocess(modifiedNodeIdsReversed, onCycle, operatingFunction) } + /** + * Returns true if the node was reached by this pass and has not been finalized yet, + * i.e., its value may still change later in the pass. + */ + public isPending(id: number): boolean { + return Boolean(this.inSCC[id]) && !this.finalized[id] + } + + /** + * Requests that a pending node is computed before the node being processed is retried. Can be called several times + * for one evaluation, e.g., for every pending cell of a range. The operating function calls it right before it + * returns {@link NodeProcessingResult.DEFERRED}. + */ + public requestPull(id: number): void { + this.pullRequests.push(id) + } + /** * Returns adjacent nodes of a given node. */ @@ -153,37 +205,168 @@ export class TopSort { } /** - * Postprocesses the result of Tarjan's algorithm. + * Postprocesses the result of Tarjan's algorithm: processes the nodes in topological order. + * + * A node whose operating function returns DEFERRED has read a pending node P. P is pulled: its pending precedents + * and then P itself are processed right away, from an explicit stack, and the node is retried. The main walk skips + * nodes that were already processed this way. A pull that reaches a node already on the pull path is a cycle. */ - private postprocess(modifiedNodeIds: number[], onCycle: (node: T) => void, operatingFunction: (node: T) => boolean) { - const shouldBeUpdatedMapping: boolean[] = [] - + private postprocess(modifiedNodeIds: number[], onCycle: (node: T) => void, operatingFunction: (node: T) => NodeProcessingResult): TopSortResult { modifiedNodeIds.forEach((t: number) => { - shouldBeUpdatedMapping[t] = true + this.shouldBeUpdatedMapping[t] = true }) - const sorted: T[] = [] - const cycled: T[] = [] this.order.reverse() - this.order.forEach((t: number) => { - const adjacentNodes = this.getAdjacentNodeIds(t) - - // The following line is a potential performance bottleneck. - // Array.includes() is O(n) operation, which makes the whole algorithm O(n^2). - // Idea for improvement: use Set[] instead of number[][] for edgesSparseArray. - if (this.sccNonSingletons[t] || adjacentNodes.includes(t)) { - cycled.push(this.nodesSparseArray[t]) - onCycle(this.nodesSparseArray[t]) - adjacentNodes.forEach((s: number) => shouldBeUpdatedMapping[s] = true) - } else { - sorted.push(this.nodesSparseArray[t]) - if (shouldBeUpdatedMapping[t] && operatingFunction(this.nodesSparseArray[t])) { - adjacentNodes.forEach((s: number) => shouldBeUpdatedMapping[s] = true) + for (const t of this.order) { + if (this.finalized[t] || this.nodesSparseArray[t] === undefined) { + continue + } + this.pullRequests = [] + if (this.processNode(t, onCycle, operatingFunction) === NodeProcessingResult.DEFERRED) { + this.pullFor(t, onCycle, operatingFunction) + } + } + + return {sorted: this.sorted, cycled: this.cycled, deferred: []} + } + + /** + * Called after the root node was deferred: computes the pending nodes it requested and retries it until it is + * finalized. Nodes on the stack form a chain: each one waits for the one above it, so reaching a node already on + * the stack is a cycle. + */ + private pullFor(root: number, onCycle: (node: T) => void, operatingFunction: (node: T) => NodeProcessingResult): void { + const stack: number[] = [root] + this.onPullPath[root] = true + this.recordRequests(root) + + while (stack.length > 0) { + const t = stack[stack.length - 1] + + if (this.finalized[t] || this.nodesSparseArray[t] === undefined) { + stack.pop() + this.onPullPath[t] = false + this.requested.delete(t) + continue + } + + const next = this.nextRequested(t) ?? (t !== root ? this.nextPendingPrecedent(t) : undefined) + if (next !== undefined) { + if (this.onPullPath[next]) { + this.finalizeCycle(stack, next, onCycle) + } else { + stack.push(next) + this.onPullPath[next] = true } + continue } - }) - return {sorted, cycled} + this.pullRequests = [] + if (this.processNode(t, onCycle, operatingFunction) === NodeProcessingResult.DEFERRED) { + this.recordRequests(t) + } + } + } + + /** + * Stores the nodes a just-deferred node requested, to be computed one by one before it is retried. + */ + private recordRequests(t: number): void { + const pending = [...new Set(this.pullRequests)].filter(id => this.isPending(id)) + if (pending.length === 0) { + throw new Error('A node was deferred without requesting a pending node.') + } + this.requested.set(t, pending.reverse()) + } + + /** + * Returns a node the given node requested that is still pending, or undefined if there is none. + */ + private nextRequested(t: number): number | undefined { + const requested = this.requested.get(t) + if (requested === undefined) { + return undefined + } + while (requested.length > 0 && !this.isPending(requested[requested.length - 1])) { + requested.pop() + } + if (requested.length === 0) { + this.requested.delete(t) + return undefined + } + return requested[requested.length - 1] + } + + /** + * Returns a precedent of a pulled node that is still pending, or undefined if there is none. + * The pending precedents are collected once per node and consumed as they get finalized. + */ + private nextPendingPrecedent(t: number): number | undefined { + let pending = this.pendingPrecedents.get(t) + if (pending === undefined) { + pending = this.precedentIds(t).filter(id => this.isPending(id)) + this.pendingPrecedents.set(t, pending) + } + while (pending.length > 0 && !this.isPending(pending[pending.length - 1])) { + pending.pop() + } + return pending.length > 0 ? pending[pending.length - 1] : undefined + } + + /** + * Gives every node on the pull path from the given node to the top of the stack the cycle treatment. + */ + private finalizeCycle(stack: number[], from: number, onCycle: (node: T) => void): void { + for (let i = stack.indexOf(from); i < stack.length; i++) { + const t = stack[i] + if (this.finalized[t]) { + continue + } + const node = this.nodesSparseArray[t] + this.cycled.push(node) + onCycle(node) + this.getAdjacentNodeIds(t).forEach((s: number) => this.shouldBeUpdatedMapping[s] = true) + this.finalized[t] = true + } + } + + /** + * Processes a single node: gives it the cycle treatment, skips it if it is current, or runs the operating function. + * Finalizes the node unless the operating function returns DEFERRED. + */ + private processNode(t: number, onCycle: (node: T) => void, operatingFunction: (node: T) => NodeProcessingResult): NodeProcessingResult { + const node = this.nodesSparseArray[t] + const adjacentNodes = this.getAdjacentNodeIds(t) + + // The following line is a potential performance bottleneck. + // Array.includes() is O(n) operation, which makes the whole algorithm O(n^2). + // Idea for improvement: use Set[] instead of number[][] for edgesSparseArray. + if (this.sccNonSingletons[t] || adjacentNodes.includes(t)) { + this.cycled.push(node) + onCycle(node) + adjacentNodes.forEach((s: number) => this.shouldBeUpdatedMapping[s] = true) + this.finalized[t] = true + return NodeProcessingResult.CHANGED + } + + if (!this.shouldBeUpdatedMapping[t]) { + this.sorted.push(node) + this.finalized[t] = true + return NodeProcessingResult.UNCHANGED + } + + const result = operatingFunction(node) + + if (result === NodeProcessingResult.DEFERRED) { + return result + } + + this.sorted.push(node) + this.finalized[t] = true + if (result === NodeProcessingResult.CHANGED) { + adjacentNodes.forEach((s: number) => this.shouldBeUpdatedMapping[s] = true) + } + return result } } diff --git a/hyperformula/src/EvaluationPass.ts b/hyperformula/src/EvaluationPass.ts new file mode 100644 index 0000000000..172bb4fea5 --- /dev/null +++ b/hyperformula/src/EvaluationPass.ts @@ -0,0 +1,102 @@ +/** + * @license + * Copyright (c) 2025 Handsoncode. All rights reserved. + */ + +import {AbsoluteCellRange, simpleCellRange} from './AbsoluteCellRange' +import {SimpleCellAddress} from './Cell' +import {CellVertex, DependencyGraph, RangeVertex, Vertex} from './DependencyGraph' +import {FormulaVertex, RuntimeDependency} from './DependencyGraph/FormulaVertex' +import {TopSortPass} from './DependencyGraph/Graph' + +const NO_DEPENDENCIES: RuntimeDependency[] = [] + +/** + * Context of the formula evaluations in one top-sort pass. + * Lets functions that discover references at runtime (e.g., INDIRECT) record them and check whether a cell is current. + */ +export class EvaluationPass { + private currentDependent?: FormulaVertex + private currentRecorded?: RuntimeDependency[] + + constructor( + private readonly dependencyGraph: DependencyGraph, + private readonly topSortPass: TopSortPass, + ) { + } + + /** + * The formula being evaluated, or undefined between evaluations. + */ + public get dependent(): FormulaVertex | undefined { + return this.currentDependent + } + + /** + * Starts recording runtime dependencies for an evaluation of the given formula. + */ + public begin(dependent: FormulaVertex): void { + this.currentDependent = dependent + this.currentRecorded = undefined + } + + /** + * Ends the current evaluation and returns the runtime dependencies it recorded, in reading order, without duplicates. + */ + public end(): RuntimeDependency[] { + const recorded = this.currentRecorded ?? NO_DEPENDENCIES + this.currentDependent = undefined + this.currentRecorded = undefined + return recorded + } + + /** + * Returns true if the vertex may still be recomputed later in this pass, i.e., its value is not current yet. + */ + public isPending(vertex: Vertex): boolean { + return this.topSortPass.isPending(vertex) + } + + /** + * Requests that a pending vertex is computed before the evaluated formula is retried. + * Call it right before throwing {@link EVALUATION_DEFERRED}. + */ + public requestPull(vertex: Vertex): void { + this.topSortPass.requestPull(vertex) + } + + /** + * Records that the evaluated formula reads the given range, and adds an edge from the range's vertex to the graph, + * creating the range vertex if needed. + * + * @returns the range vertex + */ + public recordRangeDependency(range: AbsoluteCellRange): RangeVertex { + if (this.currentDependent === undefined) { + throw new Error('No formula is being evaluated.') + } + const vertex = this.dependencyGraph.addRuntimeRangeDependency(this.currentDependent, range) + this.currentRecorded = this.currentRecorded ?? [] + if (!this.currentRecorded.some(dependency => dependency.vertex === vertex)) { + this.currentRecorded.push({address: simpleCellRange(range.start, range.end), vertex}) + } + return vertex + } + + /** + * Records that the evaluated formula reads the cell at the given address, and adds the corresponding edge to the graph. + * + * @returns the vertex of the cell that was read + */ + public recordDependency(address: SimpleCellAddress): CellVertex { + if (this.currentDependent === undefined) { + throw new Error('No formula is being evaluated.') + } + const vertex = this.dependencyGraph.addRuntimeDependency(this.currentDependent, address) + this.currentRecorded = this.currentRecorded ?? [] + if (!this.currentRecorded.some(dependency => dependency.vertex === vertex)) { + this.currentRecorded.push({address, vertex}) + } + return vertex + } +} diff --git a/hyperformula/src/Evaluator.ts b/hyperformula/src/Evaluator.ts index f810bee363..939451f64e 100644 --- a/hyperformula/src/Evaluator.ts +++ b/hyperformula/src/Evaluator.ts @@ -10,12 +10,17 @@ import {Config} from './Config' import {ContentChanges} from './ContentChanges' import {ArrayFormulaVertex, DependencyGraph, RangeVertex, Vertex} from './DependencyGraph' import {FormulaVertex} from './DependencyGraph/FormulaVertex' +import {TopSortPass} from './DependencyGraph/Graph' +import {NodeProcessingResult} from './DependencyGraph/TopSort' +import {EvaluationPass} from './EvaluationPass' +import {EVALUATION_DEFERRED} from './interpreter/EvaluationDeferred' import {Interpreter} from './interpreter/Interpreter' import {InterpreterState} from './interpreter/InterpreterState' import {EmptyValue, getRawValue, InterpreterValue} from './interpreter/InterpreterValue' import {SimpleRangeValue} from './SimpleRangeValue' import {LazilyTransformingAstService} from './LazilyTransformingAstService' import {ColumnSearchStrategy} from './Lookup/SearchStrategy' +import {Maybe} from './Maybe' import {Ast, RelativeDependency} from './parser' import {Statistics, StatType} from './statistics' @@ -32,12 +37,11 @@ export class Evaluator { } public run(): void { - this.stats.start(StatType.TOP_SORT) - const {sorted, cycled} = this.dependencyGraph.topSortWithScc() - this.stats.end(StatType.TOP_SORT) - this.stats.measure(StatType.EVALUATION, () => { - this.recomputeFormulas(cycled, sorted) + this.evaluateInPasses(this.dependencyGraph.graph.getNodes(), + (vertex: Vertex, pass: EvaluationPass) => this.computeVertexOnInitialRun(vertex, pass), + (vertex: Vertex) => this.setCycleErrorOnInitialRun(vertex), + ) }) } @@ -45,8 +49,8 @@ export class Evaluator { const changes = ContentChanges.empty() this.stats.measure(StatType.EVALUATION, () => { - this.dependencyGraph.graph.getTopSortedWithSccSubgraphFrom(vertices, - (vertex: Vertex) => this.recomputeVertex(vertex, changes), + this.evaluateInPasses(vertices, + (vertex: Vertex, pass: EvaluationPass) => this.recomputeVertex(vertex, changes, pass), (vertex: Vertex) => this.processVertexOnCycle(vertex, changes), ) }) @@ -74,25 +78,98 @@ export class Evaluator { return ret } + /** + * Evaluates the graph starting from the seeds. + * + * A formula that reads a cell which is not current yet (see {@link EVALUATION_DEFERRED}) requests that cell. The sort + * computes the cell, and the pending cells it depends on, right away and then retries the formula, all in the same pass. + * + * A formula on a cycle loses its runtime dependencies and is recomputed once more, in another pass, so that a formula + * that no longer reads a cell on the cycle can leave it. If it reads one again, it keeps #CYCLE!. + */ + private evaluateInPasses( + seeds: Vertex[], + operatingFunction: (vertex: Vertex, pass: EvaluationPass) => NodeProcessingResult, + onCycle: (vertex: Vertex) => void, + ): void { + const retriedAfterCycle = new Set() + let nextSeeds = seeds + + while (nextSeeds.length > 0) { + const retrySeeds: Vertex[] = [] + let evaluationPass: Maybe = undefined + const {deferred} = this.dependencyGraph.graph.getTopSortedWithSccSubgraphFrom(nextSeeds, + (vertex: Vertex, topSortPass: TopSortPass) => { + evaluationPass = evaluationPass ?? new EvaluationPass(this.dependencyGraph, topSortPass) + return operatingFunction(vertex, evaluationPass) + }, + (vertex: Vertex) => { + onCycle(vertex) + if (vertex instanceof FormulaVertex && !retriedAfterCycle.has(vertex) && this.dependencyGraph.clearRuntimeDependencies(vertex)) { + retriedAfterCycle.add(vertex) + retrySeeds.push(vertex) + } + }, + ) + nextSeeds = [...deferred, ...retrySeeds] + } + this.dependencyGraph.removeUnusedRuntimeVertices() + } + + /** + * Computes the value of a vertex during the initial evaluation of the whole graph. + */ + private computeVertexOnInitialRun(vertex: Vertex, pass: EvaluationPass): NodeProcessingResult { + if (vertex instanceof FormulaVertex) { + const address = vertex.getAddress(this.lazilyTransformingAstService) + const previousValue = vertex.valueOrUndef() + const newCellValue = this.recomputeFormulaVertexValue(vertex, address, pass) + if (newCellValue === undefined) { + return NodeProcessingResult.DEFERRED + } + if (previousValue === undefined) { + this.columnSearch.add(getRawValue(newCellValue), address) + } else { + // recomputed in a retry pass after a cycle + this.columnSearch.change(getRawValue(previousValue), getRawValue(newCellValue), address) + } + } else if (vertex instanceof RangeVertex) { + vertex.clearCache() + } + return NodeProcessingResult.CHANGED + } + + /** + * Sets the cycle error on a vertex that is part of a cycle, during the initial evaluation of the whole graph. + */ + private setCycleErrorOnInitialRun(vertex: Vertex): void { + if (vertex instanceof FormulaVertex) { + vertex.setCellValue(new CellError(ErrorType.CYCLE, undefined, vertex)) + } + } + /** * Recalculates the value of a single vertex assuming its dependencies have already been recalculated */ - private recomputeVertex(vertex: Vertex, changes: ContentChanges): boolean { + private recomputeVertex(vertex: Vertex, changes: ContentChanges, pass: EvaluationPass): NodeProcessingResult { if (vertex instanceof FormulaVertex) { + const address = vertex.getAddress(this.lazilyTransformingAstService) const currentValue = vertex.isComputed() ? vertex.getCellValue() : undefined - const newCellValue = this.recomputeFormulaVertexValue(vertex) + const newCellValue = this.recomputeFormulaVertexValue(vertex, address, pass) + if (newCellValue === undefined) { + return NodeProcessingResult.DEFERRED + } if (newCellValue !== currentValue) { - const address = vertex.getAddress(this.lazilyTransformingAstService) changes.addChange(newCellValue, address) this.columnSearch.change(getRawValue(currentValue), getRawValue(newCellValue), address) - return true + return NodeProcessingResult.CHANGED } - return false + return NodeProcessingResult.UNCHANGED } else if (vertex instanceof RangeVertex) { vertex.clearCache() - return true + return NodeProcessingResult.CHANGED } else { - return true + return NodeProcessingResult.CHANGED } } @@ -112,39 +189,41 @@ export class Evaluator { } /** - * Recalculates formulas in the topological sort order + * Evaluates a formula vertex and stores its value and runtime dependencies. + * + * @returns the new value, or undefined if the evaluation was deferred to a later pass */ - private recomputeFormulas(cycled: Vertex[], sorted: Vertex[]): void { - cycled.forEach((vertex: Vertex) => { - if (vertex instanceof FormulaVertex) { - vertex.setCellValue(new CellError(ErrorType.CYCLE, undefined, vertex)) - } - }) - sorted.forEach((vertex: Vertex) => { - if (vertex instanceof FormulaVertex) { - const newCellValue = this.recomputeFormulaVertexValue(vertex) - const address = vertex.getAddress(this.lazilyTransformingAstService) - this.columnSearch.add(getRawValue(newCellValue), address) - } else if (vertex instanceof RangeVertex) { - vertex.clearCache() - } - }) - } - - private recomputeFormulaVertexValue(vertex: FormulaVertex): InterpreterValue { - const address = vertex.getAddress(this.lazilyTransformingAstService) + private recomputeFormulaVertexValue(vertex: FormulaVertex, address: SimpleCellAddress, evaluationPass: EvaluationPass): Maybe { if (vertex instanceof ArrayFormulaVertex && (vertex.array.size.isRef || !this.dependencyGraph.isThereSpaceForArray(vertex))) { + this.dependencyGraph.clearRuntimeDependencies(vertex) return vertex.setNoSpace() - } else { - const formula = vertex.getFormula(this.lazilyTransformingAstService) - const newCellValue = this.evaluateAstToCellValue(formula, new InterpreterState(address, this.config.useArrayArithmetic, vertex)) - return vertex.setCellValue(newCellValue) } + + const formula = vertex.getFormula(this.lazilyTransformingAstService) + let newCellValue: InterpreterValue + evaluationPass.begin(vertex) + this.interpreter.evaluationPass = evaluationPass + try { + newCellValue = this.evaluateAstToCellValue(formula, new InterpreterState(address, this.config.useArrayArithmetic, vertex)) + } catch (e) { + this.dependencyGraph.keepRuntimeDependencies(vertex, evaluationPass.end()) + if (e === EVALUATION_DEFERRED) { + return undefined + } + throw e + } finally { + this.interpreter.evaluationPass = undefined + } + + this.dependencyGraph.commitRuntimeDependencies(vertex, evaluationPass.end()) + return vertex.setCellValue(newCellValue) } private evaluateAstToCellValue(ast: Ast, state: InterpreterState): InterpreterValue { const interpreterValue = this.interpreter.evaluateAst(ast, state) if (interpreterValue instanceof SimpleRangeValue) { + // Formulas that read this cell later read the range's cells, so they become dependencies of this formula now + interpreterValue.runBeforeRead() return interpreterValue } else if (interpreterValue === EmptyValue && this.config.evaluateNullToZero) { return 0 diff --git a/hyperformula/src/GraphBuilder.ts b/hyperformula/src/GraphBuilder.ts index 6d54d0e54e..9d94204a76 100644 --- a/hyperformula/src/GraphBuilder.ts +++ b/hyperformula/src/GraphBuilder.ts @@ -112,6 +112,12 @@ export class SimpleStrategy implements GraphBuilderStrategy { const vertex = new ArrayFormulaVertex(parseResult.ast, address, new ArraySize(size.width, size.height)) dependencies.set(vertex, absolutizeDependencies(parseResult.dependencies, address)) this.dependencyGraph.addArrayVertex(address, vertex) + if (parseResult.hasVolatileFunction) { + this.dependencyGraph.markAsVolatile(vertex) + } + if (parseResult.hasStructuralChangeFunction) { + this.dependencyGraph.markAsDependentOnStructureChange(vertex) + } } } } else if (parsedCellContent instanceof CellContent.Empty) { diff --git a/hyperformula/src/Operations.ts b/hyperformula/src/Operations.ts index df6d41264f..9d883ac352 100644 --- a/hyperformula/src/Operations.ts +++ b/hyperformula/src/Operations.ts @@ -284,6 +284,8 @@ export class Operations { mergedPlaceholderSheetId?: number, } { const { previousDisplayName, mergedWithPlaceholderSheet } = this.sheetMapping.renameSheet(sheetId, newName) + // a formula like INDIRECT("OldName!A1") must resolve again + this.dependencyGraph.markStructureDependentFormulasAsDirty() let version: number | undefined if (mergedWithPlaceholderSheet !== undefined) { @@ -421,6 +423,8 @@ export class Operations { const namedExpression = this.namedExpressions.addNamedExpression(expressionName, sheetId, options) this.storeNamedExpressionInCell(namedExpression.address, expression) this.adjustNamedExpressionEdges(namedExpression, expressionName, sheetId) + // a formula like INDIRECT("Name") may now resolve + this.dependencyGraph.markStructureDependentFormulasAsDirty() } public restoreNamedExpression(namedExpression: InternalNamedExpression, content: ClipboardCell, sheetId?: number) { @@ -428,6 +432,7 @@ export class Operations { this.restoreCell(namedExpression.address, content) const restoredNamedExpression = this.namedExpressions.restoreNamedExpression(namedExpression, sheetId) this.adjustNamedExpressionEdges(restoredNamedExpression, expressionName, sheetId) + this.dependencyGraph.markStructureDependentFormulasAsDirty() } public changeNamedExpressionExpression(expressionName: string, newExpression: RawCellContent, sheetId?: number, options?: NamedExpressionOptions): [InternalNamedExpression, ClipboardCell] { @@ -441,6 +446,8 @@ export class Operations { namedExpression.options = options const content = this.getClipboardCell(namedExpression.address) this.storeNamedExpressionInCell(namedExpression.address, newExpression) + // a formula like INDIRECT("Name") must resolve again + this.dependencyGraph.markStructureDependentFormulasAsDirty() return [oldNamedExpression, content] } @@ -450,6 +457,7 @@ export class Operations { throw new NamedExpressionDoesNotExistError(expressionName) } this.namedExpressions.remove(namedExpression.displayName, sheetId) + this.dependencyGraph.markStructureDependentFormulasAsDirty() const content = this.getClipboardCell(namedExpression.address) if (sheetId !== undefined) { const globalNamedExpression = this.namedExpressions.workbookNamedExpressionOrPlaceholder(expressionName) diff --git a/hyperformula/src/SimpleRangeValue.ts b/hyperformula/src/SimpleRangeValue.ts index e7af429b2b..ea515ba590 100644 --- a/hyperformula/src/SimpleRangeValue.ts +++ b/hyperformula/src/SimpleRangeValue.ts @@ -20,25 +20,38 @@ export class SimpleRangeValue { */ public readonly size: ArraySize + private readonly _range?: AbsoluteCellRange + + /** + * Runs before the range's cells are first read, e.g., to make sure they are computed. May throw to defer the + * evaluation that reads them; it runs again on the next read. + */ + private beforeRead?: () => void + /** * In most cases, it's more convenient to create a `SimpleRangeValue` object * by calling one of the [static factory methods](#fromrange). */ constructor( private _data?: InternalScalarValue[][], - - /** - * A property that represents the address of the range. - */ - public readonly range?: AbsoluteCellRange, + range?: AbsoluteCellRange, private readonly dependencyGraph?: DependencyGraph, private _hasOnlyNumbers?: boolean, ) { + this._range = range this.size = _data === undefined ? new ArraySize(range!.effectiveWidth(dependencyGraph!), range!.effectiveHeight(dependencyGraph!)) : new ArraySize(_data[0].length, _data.length) } + /** + * A property that represents the address of the range. + */ + public get range(): AbsoluteCellRange | undefined { + this.runBeforeRead() + return this._range + } + /** * Returns the range data as a 2D array. */ @@ -75,6 +88,18 @@ export class SimpleRangeValue { return new SimpleRangeValue(undefined, range, dependencyGraph, undefined) } + /** + * A factory method. Like {@link onlyRange}, but runs the given function before the range's cells or address are + * first used (its size can be read without it). Used for ranges that functions discover at runtime, e.g., INDIRECT. + * + * @internal + */ + public static onlyRangeWithReadHook(range: AbsoluteCellRange, dependencyGraph: DependencyGraph, beforeRead: () => void): SimpleRangeValue { + const value = new SimpleRangeValue(undefined, range, dependencyGraph, undefined) + value.beforeRead = beforeRead + return value + } + /** * A factory method. Returns a `SimpleRangeValue` object that contains a single value. */ @@ -86,7 +111,7 @@ export class SimpleRangeValue { * Returns `true` if and only if the `SimpleRangeValue` has no address set. */ public isAdHoc(): boolean { - return this.range === undefined + return this._range === undefined } /** @@ -208,13 +233,26 @@ export class SimpleRangeValue { /** * Computes the range data if it is not computed yet. */ + /** + * Runs the function given to {@link onlyRangeWithReadHook} unless it has already run. + * + * @internal + */ + public runBeforeRead(): void { + if (this.beforeRead !== undefined) { + this.beforeRead() + this.beforeRead = undefined + } + } + private ensureThatComputed(): void { if (this._data !== undefined) { return } + this.runBeforeRead() this._hasOnlyNumbers = true - this._data = this.range!.addressesArrayMap(this.dependencyGraph!, cellFromRange => { + this._data = this._range!.addressesArrayMap(this.dependencyGraph!, cellFromRange => { const value = this.dependencyGraph!.getCellValue(cellFromRange) if (value instanceof SimpleRangeValue) { this._hasOnlyNumbers = false diff --git a/hyperformula/src/i18n/languages/enGB.ts b/hyperformula/src/i18n/languages/enGB.ts index d271b732cd..b5ef33e0de 100644 --- a/hyperformula/src/i18n/languages/enGB.ts +++ b/hyperformula/src/i18n/languages/enGB.ts @@ -123,6 +123,7 @@ const dictionary: RawTranslationPackage = { IFNA: 'IFNA', IFS: 'IFS', INDEX: 'INDEX', + INDIRECT: 'INDIRECT', INT: 'INT', INTERVAL: 'INTERVAL', IPMT: 'IPMT', diff --git a/hyperformula/src/interpreter/EvaluationDeferred.ts b/hyperformula/src/interpreter/EvaluationDeferred.ts new file mode 100644 index 0000000000..1cab83402b --- /dev/null +++ b/hyperformula/src/interpreter/EvaluationDeferred.ts @@ -0,0 +1,20 @@ +/** + * @license + * Copyright (c) 2025 Handsoncode. All rights reserved. + */ + +/** + * Signal thrown by a function that needs the value of a cell that is not current yet in the running evaluation pass. + * The evaluator catches it and recomputes the formula in a later pass. Plugins must not catch it. + */ +export class EvaluationDeferred extends Error { + constructor() { + super('Evaluation deferred to a later pass.') + this.name = 'EvaluationDeferred' + } +} + +/** + * The only instance of {@link EvaluationDeferred}. Created once, so throwing it does not capture a stack trace each time. + */ +export const EVALUATION_DEFERRED = new EvaluationDeferred() diff --git a/hyperformula/src/interpreter/Interpreter.ts b/hyperformula/src/interpreter/Interpreter.ts index 6444332537..b4056ff82d 100644 --- a/hyperformula/src/interpreter/Interpreter.ts +++ b/hyperformula/src/interpreter/Interpreter.ts @@ -8,6 +8,7 @@ import {ArraySizePredictor} from '../ArraySize' import {ArrayValue, NotComputedArray} from '../ArrayValue' import {CellError, ErrorType, isColOrRowInvalid} from '../Cell' import {Config} from '../Config' +import {EvaluationPass} from '../EvaluationPass' import {DateTimeHelper} from '../DateTimeHelper' import {DependencyGraph} from '../DependencyGraph' import {FormulaVertex} from '../DependencyGraph/FormulaVertex' @@ -44,6 +45,11 @@ import { AddressWithSheet } from '../parser/Address' export class Interpreter { public readonly criterionBuilder: CriterionBuilder + /** + * Context of the formula evaluation in progress, set by the Evaluator. Undefined outside of evaluation passes + * (e.g., in calculateFormula). + */ + public evaluationPass?: EvaluationPass /** Decides which function calls the license stops; see {@link FunctionCallLicenseGate}. */ private readonly functionCallLicenseGate: FunctionCallLicenseGate diff --git a/hyperformula/src/interpreter/functionMetadata/categories/lookup-and-reference.ts b/hyperformula/src/interpreter/functionMetadata/categories/lookup-and-reference.ts index ef6b848d7f..5dee161eb0 100644 --- a/hyperformula/src/interpreter/functionMetadata/categories/lookup-and-reference.ts +++ b/hyperformula/src/interpreter/functionMetadata/categories/lookup-and-reference.ts @@ -81,6 +81,13 @@ export const LOOKUP_AND_REFERENCE_DOCS: Record = { documentationUrl: 'https://hyperformula.handsontable.com/docs/guide/built-in-functions.html', examples: ['=INDEX(A1:C10, 2, 3)', '=INDEX(A1:A10, 5)'], }, + INDIRECT: { + category: 'Lookup and reference', + shortDescription: 'Returns the value of the cell whose address is given as text. Only single-cell A1 addresses are supported.', + parameters: [{name: 'ref_text', description: 'The address of a single cell as text, in A1 notation, optionally with a sheet name.'}, {name: 'a1', description: 'TRUE (default) for A1 notation. FALSE (R1C1 notation) is not supported and returns #REF!.'}], + documentationUrl: 'https://hyperformula.handsontable.com/docs/guide/built-in-functions.html', + examples: ['=INDIRECT("B1")', '=INDIRECT("Sheet2!$A$1")'], + }, MATCH: { category: 'Lookup and reference', shortDescription: 'Returns the relative position of an item in an array that matches a specified value.', diff --git a/hyperformula/src/interpreter/plugin/IndirectPlugin.ts b/hyperformula/src/interpreter/plugin/IndirectPlugin.ts new file mode 100644 index 0000000000..fafa13f153 --- /dev/null +++ b/hyperformula/src/interpreter/plugin/IndirectPlugin.ts @@ -0,0 +1,248 @@ +/** + * @license + * Copyright (c) 2025 Handsoncode. All rights reserved. + */ + +import {AbsoluteCellRange, SimpleCellRange, simpleCellRange} from '../../AbsoluteCellRange' +import {CellError, ErrorType, simpleCellAddress, SimpleCellAddress} from '../../Cell' +import {Vertex} from '../../DependencyGraph' +import {FormulaVertex} from '../../DependencyGraph/FormulaVertex' +import {ErrorMessage} from '../../error-message' +import {Maybe} from '../../Maybe' +import {AstNodeType, ProcedureAst} from '../../parser' +import {simpleCellAddressFromString, simpleCellRangeFromString} from '../../parser/addressRepresentationConverters' +import {SimpleRangeValue} from '../../SimpleRangeValue' +import {EVALUATION_DEFERRED} from '../EvaluationDeferred' +import {InterpreterState} from '../InterpreterState' +import {InterpreterValue} from '../InterpreterValue' +import {FunctionArgumentType, FunctionPlugin, FunctionPluginTypecheck, ImplementedFunctions} from './FunctionPlugin' + +const R1C1_CELL = /^R(\[-?\d+\]|\d+)?C(\[-?\d+\]|\d+)?$/i + +/** + * Parses one R1C1 cell: R and C followed by an absolute 1-based number, a relative [offset], or nothing (offset 0). + */ +function r1c1CellAddress(text: string, sheet: number, formulaAddress: SimpleCellAddress): Maybe { + const match = R1C1_CELL.exec(text) + if (match === null) { + return undefined + } + const coordinate = (part: string | undefined, base: number): number => { + if (part === undefined) { + return base + } + if (part.startsWith('[')) { + return base + Number(part.slice(1, -1)) + } + return Number(part) - 1 + } + return simpleCellAddress(sheet, coordinate(match[2], formulaAddress.col), coordinate(match[1], formulaAddress.row)) +} + +/** + * Removes the quotes around a sheet name like 'My Sheet', where '' stands for one quote. + */ +function unquoteSheetName(name: string): string { + if (name.length >= 2 && name.startsWith("'") && name.endsWith("'")) { + return name.slice(1, -1).replace(/''/g, "'") + } + return name +} + +export class IndirectPlugin extends FunctionPlugin implements FunctionPluginTypecheck { + public static implementedFunctions: ImplementedFunctions = { + 'INDIRECT': { + method: 'indirect', + parameters: [ + {argumentType: FunctionArgumentType.STRING}, + {argumentType: FunctionArgumentType.BOOLEAN, optionalArg: true, defaultValue: true, emptyAsDefault: true}, + ], + // Not volatile: a formula is recomputed through the edges to its selector and to the cell it read. Edits that + // change what an address points to without changing a cell (structure, sheets) mark it dirty instead. + isDependentOnSheetStructureChange: true, + vectorizationForbidden: true, + }, + } + + /** + * Corresponds to INDIRECT(ref_text, [a1]) + * + * Returns the value of the cell, or the range, whose address is given as text. Supports A1 addresses of single + * cells and of ranges, optionally with a sheet name, and names of named expressions that refer to a cell or range. + * With a1 = FALSE, the text is an R1C1 reference to a cell or range, or a name. + * + * @param ast + * @param state + */ + public indirect(ast: ProcedureAst, state: InterpreterState): InterpreterValue { + return this.runFunction(ast.args, state, this.metadata('INDIRECT'), (refText: string, a1: boolean): InterpreterValue => { + const resolveSheet = (sheetName: string) => this.dependencyGraph.sheetMapping.getSheetId(sheetName) + + if (!a1) { + const range = this.r1c1Reference(refText, state.formulaAddress) ?? this.namedReference(refText, state.formulaAddress.sheet) + return this.readReference(range, state) + } + + const address = simpleCellAddressFromString(resolveSheet, refText, state.formulaAddress.sheet) + + if (address !== undefined) { + if (!this.isExistingAddress(address)) { + return new CellError(ErrorType.REF, ErrorMessage.BadRef) + } + return this.readCell(address, state) + } + + const range = simpleCellRangeFromString(resolveSheet, refText, state.formulaAddress.sheet) + ?? this.namedReference(refText, state.formulaAddress.sheet) + return this.readReference(range, state) + }) + } + + /** + * Reads a reference given by its two corners: a cell if they are the same, otherwise a range. + * Returns #REF! if there is no reference or it leaves the existing sheets or the sheet limits. + */ + private readReference(range: Maybe, state: InterpreterState): InterpreterValue { + if (range === undefined || !this.isExistingAddress(range.start) || !this.isExistingAddress(range.end)) { + return new CellError(ErrorType.REF, ErrorMessage.BadRef) + } + if (range.start.row === range.end.row && range.start.col === range.end.col) { + return this.readCell(range.start, state) + } + // corners in any order, like Excel: "B2:A1" is A1:B2 + const normalized = AbsoluteCellRange.fromCoordinates(range.start.sheet, + Math.min(range.start.col, range.end.col), Math.min(range.start.row, range.end.row), + Math.max(range.start.col, range.end.col), Math.max(range.start.row, range.end.row)) + return this.readRange(normalized, state) + } + + /** + * Parses an R1C1 reference to a cell or a range of cells, optionally with a sheet name: R2C3, R[-1]C[2] + * (relative to the formula's cell), RC (the formula's own cell), R1C1:R2C2. Whole rows and columns are not + * supported. Returns undefined if the text is not such a reference or names an unknown sheet. + */ + private r1c1Reference(text: string, formulaAddress: SimpleCellAddress): Maybe { + const sheetSeparator = text.lastIndexOf('!') + let sheet = formulaAddress.sheet + let reference = text + if (sheetSeparator >= 0) { + const sheetId = this.dependencyGraph.sheetMapping.getSheetId(unquoteSheetName(text.slice(0, sheetSeparator))) + if (sheetId === undefined) { + return undefined + } + sheet = sheetId + reference = text.slice(sheetSeparator + 1) + } + const corners = reference.split(':') + if (corners.length > 2) { + return undefined + } + const start = r1c1CellAddress(corners[0], sheet, formulaAddress) + const end = corners.length === 2 ? r1c1CellAddress(corners[1], sheet, formulaAddress) : start + if (start === undefined || end === undefined) { + return undefined + } + return simpleCellRange(start, end) + } + + /** + * Returns the cell or range a named expression refers to, as a range (a single cell is a one-cell range), + * or undefined if there is no such name in scope or it is not a reference (e.g., =5). + */ + private namedReference(name: string, sheet: number): Maybe { + const namedExpression = this.dependencyGraph.namedExpressions.nearestNamedExpression(name, sheet) + if (namedExpression === undefined) { + return undefined + } + const vertex = this.dependencyGraph.addressMapping.getCell(namedExpression.address) + if (!(vertex instanceof FormulaVertex)) { + return undefined + } + const ast = vertex.getFormula(this.dependencyGraph.lazilyTransformingAstService) + const baseAddress = vertex.getAddress(this.dependencyGraph.lazilyTransformingAstService) + if (ast.type === AstNodeType.CELL_REFERENCE) { + const address = ast.reference.toSimpleCellAddress(baseAddress) + return new AbsoluteCellRange(address, address) + } + if (ast.type === AstNodeType.CELL_RANGE) { + return AbsoluteCellRange.fromAstOrUndef(ast, baseAddress) + } + return undefined + } + + /** + * Returns true if the address is on an existing sheet and within the limits set by the maxRows and maxColumns options. + */ + private isExistingAddress(address: SimpleCellAddress): boolean { + return this.dependencyGraph.sheetMapping.hasSheetWithId(address.sheet) && this.isWithinSheetLimits(address) + } + + /** + * Returns the range as a reference. Its cells are read lazily: only when a function uses the values (SUM) or the + * address (lookups, criteria), not when it only needs the size (ROWS), so the range is recorded as a runtime + * dependency of the evaluated formula only then. Pending cells are requested and the evaluation is deferred. + */ + private readRange(range: AbsoluteCellRange, state: InterpreterState): SimpleRangeValue { + const pass = this.interpreter.evaluationPass + const dependent = state.formulaVertex + return SimpleRangeValue.onlyRangeWithReadHook(range, this.dependencyGraph, () => { + if (pass === undefined || dependent === undefined || pass.dependent !== dependent) { + return + } + const rangeVertex = pass.recordRangeDependency(range) + const pending: Vertex[] = [] + if (pass.isPending(rangeVertex)) { + pending.push(rangeVertex) + } else { + for (const address of range.addresses(this.dependencyGraph)) { + const cell = this.dependencyGraph.addressMapping.getCell(address) + if (cell !== undefined && pass.isPending(cell)) { + pending.push(cell) + } + } + } + if (pending.length > 0) { + pending.forEach(vertex => pass.requestPull(vertex)) + throw EVALUATION_DEFERRED + } + }) + } + + /** + * Returns true if the address is within the limits set by the maxRows and maxColumns options. + */ + private isWithinSheetLimits(address: SimpleCellAddress): boolean { + return address.row >= 0 && address.col >= 0 && address.row < this.config.maxRows && address.col < this.config.maxColumns + } + + /** + * Reads the value of a cell, recording it as a runtime dependency of the evaluated formula. + * If the cell is a formula that is not current yet in the running pass, requests it and defers the evaluation: + * the formula is retried once the cell is computed. + */ + private readCell(address: SimpleCellAddress, state: InterpreterState): InterpreterValue { + const pass = this.interpreter.evaluationPass + + if (pass === undefined || state.formulaVertex === undefined || pass.dependent !== state.formulaVertex) { + return this.dependencyGraph.getCellValue(address) + } + + if (this.dependencyGraph.addressMapping.getCell(address) === state.formulaVertex) { + return new CellError(ErrorType.CYCLE, undefined, state.formulaVertex) + } + + const vertex = pass.recordDependency(address) + + if (vertex instanceof FormulaVertex) { + if (pass.isPending(vertex)) { + pass.requestPull(vertex) + throw EVALUATION_DEFERRED + } + if (!vertex.isComputed()) { + return new CellError(ErrorType.REF, ErrorMessage.BadRef) + } + } + + return this.dependencyGraph.getCellValue(address) + } +} diff --git a/hyperformula/src/interpreter/plugin/index.ts b/hyperformula/src/interpreter/plugin/index.ts index 2a3005375b..bab3da9759 100644 --- a/hyperformula/src/interpreter/plugin/index.ts +++ b/hyperformula/src/interpreter/plugin/index.ts @@ -21,6 +21,7 @@ export {ExpPlugin} from './ExpPlugin' export {FinancialPlugin} from './FinancialPlugin' export {FormulaTextPlugin} from './FormulaTextPlugin' export {HyperlinkPlugin} from './HyperlinkPlugin' +export {IndirectPlugin} from './IndirectPlugin' export {InformationPlugin} from './InformationPlugin' export {IsEvenPlugin} from './IsEvenPlugin' export {IsOddPlugin} from './IsOddPlugin' diff --git a/hyperformula/src/license/functionCapabilities.ts b/hyperformula/src/license/functionCapabilities.ts index a26733ec5f..85e0258cdc 100644 --- a/hyperformula/src/license/functionCapabilities.ts +++ b/hyperformula/src/license/functionCapabilities.ts @@ -88,7 +88,7 @@ const UNGROUPED_FUNCTIONS = [ 'GAMMA.INV', 'GAMMALN', 'GAUSS', 'GCD', 'GEOMEAN', 'HARMEAN', 'HEX2BIN', 'HEX2OCT', 'HYPGEOM.DIST', 'IMABS', 'IMAGINARY', 'IMARGUMENT', 'IMCONJUGATE', 'IMCOS', 'IMCOSH', 'IMCOT', 'IMCSC', 'IMCSCH', 'IMDIV', 'IMEXP', 'IMLN', 'IMLOG10', 'IMLOG2', 'IMPOWER', 'IMPRODUCT', 'IMREAL', 'IMSEC', 'IMSECH', 'IMSIN', - 'IMSINH', 'IMSQRT', 'IMSUB', 'IMSUM', 'IMTAN', 'INTERVAL', 'ISBINARY', 'ISFORMULA', 'ISNONTEXT', 'ISPMT', + 'IMSINH', 'IMSQRT', 'IMSUB', 'IMSUM', 'IMTAN', 'INDIRECT', 'INTERVAL', 'ISBINARY', 'ISFORMULA', 'ISNONTEXT', 'ISPMT', 'ISREF', 'LCM', 'LOG10', 'LOGNORM.DIST', 'LOGNORM.INV', 'MAXA', 'MAXPOOL', 'MEDIANPOOL', 'MINA', 'MIRR', 'MMULT', 'MULTINOMIAL', 'NEGBINOM.DIST', 'NETWORKDAYS.INTL', 'NOMINAL', 'NORM.DIST', 'NORM.INV', 'NORM.S.DIST', 'NORM.S.INV', 'NPER', 'OCT2BIN', 'OCT2DEC', 'OCT2HEX', 'PDURATION', 'PERCENTILE.EXC',