Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -531,6 +531,11 @@
"default": false,
"description": "Internal flag indicating the global references migration has completed."
},
"codex-project-manager.editValueObjectMigrationCompleted": {
"type": "boolean",
"default": false,
"description": "Internal flag indicating the edit value object migration has completed."
},
"codex-project-manager.chatSystemMessageToMetadataMigrationCompleted": {
"type": "boolean",
"default": false,
Expand Down
23 changes: 16 additions & 7 deletions sharedUtils/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -47,13 +47,22 @@ export const getCellValueData = (cell: QuillCellContent) => {
// Ensure editHistory exists and is an array
const editHistory = cell.editHistory || [];

// Find the latest edit that matches the current cell content (strict match).
// Falls back to the latest value edit if the strict match fails, which can happen
// when the merge step during save subtly normalizes the stored value.
const reversed = editHistory.slice().reverse();
const latestEditThatMatchesCellValue =
reversed.find((edit) => EditMapUtils.isValue(edit.editMap) && edit.value === cell.cellContent) ??
reversed.find((edit) => EditMapUtils.isValue(edit.editMap) && !edit.preview);
// O(1) lookup by activeEditId when available
let latestEditThatMatchesCellValue: (typeof editHistory)[number] | undefined;
if (cell.activeEditId) {
latestEditThatMatchesCellValue = editHistory.find(
(edit) => (edit as any).id === cell.activeEditId && EditMapUtils.isValue(edit.editMap)
);
}

// Fallback: find the latest edit that matches the current cell content (strict match).
// Falls back to the latest value edit if the strict match fails.
if (!latestEditThatMatchesCellValue) {
const reversed = editHistory.slice().reverse();
latestEditThatMatchesCellValue =
reversed.find((edit) => EditMapUtils.isValue(edit.editMap) && edit.value === cell.cellContent) ??
reversed.find((edit) => EditMapUtils.isValue(edit.editMap) && !edit.preview);
}

// Get audio validation from attachments instead of edits
let audioValidatedBy: ValidationEntry[] = [];
Expand Down
2 changes: 2 additions & 0 deletions src/cellLabelImporter/updater.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import * as vscode from "vscode";
import { randomUUID } from "crypto";
import { CellLabelData, CellMetadata, FileData } from "./types";
import { CodexContentSerializer } from "../serializer";
import { getNotebookMetadataManager } from "../utils/notebookMetadataManager";
Expand Down Expand Up @@ -131,6 +132,7 @@ async function saveNotebookFileWithLabels(

// Create edit history entry for the label change
cell.metadata.edits.push({
id: randomUUID(),
editMap: EditMapUtils.cellLabel(),
value: newLabel,
timestamp: currentTimestamp,
Expand Down
3 changes: 2 additions & 1 deletion src/codexMigrationTool/matcher.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import type {
import { removeHtmlTags } from "../exportHandler/subtitleUtils";
import { CodexCellTypes } from "../../types/enums";
import { isContentCell } from "../utils/cellTypeUtils";
import { resolveCellValue } from "../utils/cellValueResolver";

type SourceLine = {
cellId: string;
Expand Down Expand Up @@ -61,7 +62,7 @@ const buildSourceLinesFromFile = (file: FileData): SourceLine[] => {
}
const cellId = getCellId(cell);
if (!cellId) return;
const value = typeof cell.value === "string" ? cell.value : "";
const value = resolveCellValue(cell as any);
const normalized = normalizeText(value);
if (!normalized) return;
lines.push({ cellId, sourceValue: normalized });
Expand Down
5 changes: 4 additions & 1 deletion src/codexMigrationTool/updater.ts
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
import * as vscode from "vscode";
import { randomUUID } from "crypto";
import { CodexContentSerializer } from "../serializer";
import { mergeDuplicateCellsUsingResolverLogic } from "../projectManager/utils/merge/resolvers";
import { EditMapUtils } from "../utils/editMapUtils";
import { EditType } from "../../types/enums";
import { getAuthApi } from "../extension";
import type { CustomNotebookCellData } from "../../types";
import type { MigrationMatchResult } from "./types";
import { resolveCellValue } from "../utils/cellValueResolver";

const cloneCell = (cell: CustomNotebookCellData): CustomNotebookCellData => {
return JSON.parse(JSON.stringify(cell)) as CustomNotebookCellData;
Expand All @@ -28,8 +30,9 @@ const addMigrationEdit = (
cell.metadata.edits = [];
}
cell.metadata.edits.push({
id: randomUUID(),
editMap: EditMapUtils.value(),
value: cell.value,
value: resolveCellValue(cell as any),
timestamp,
type: EditType.MIGRATION,
author,
Expand Down
2 changes: 2 additions & 0 deletions src/extension.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import {
migration_verseRangeLabelsAndPositions,
migration_cellIdsToUuid,
migration_recoverTempFilesAndMergeDuplicates,
migration_editValueObject,
} from "./projectManager/utils/migrationUtils";
import { createIndexWithContext } from "./activationHelpers/contextAware/contentIndexes/indexes";
import { StatusBarItem } from "vscode";
Expand Down Expand Up @@ -899,6 +900,7 @@ export async function activate(context: vscode.ExtensionContext) {
await migration_addGlobalReferences(context);
await migration_cellIdsToUuid(context);
await migration_recoverTempFilesAndMergeDuplicates(context);
await migration_editValueObject(context);
}

// Remove leftover files from features that have been removed
Expand Down
32 changes: 27 additions & 5 deletions src/projectManager/utils/merge/resolvers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -943,8 +943,13 @@ function applyEditToCell(cell: CustomNotebookCellData, edit: EditHistory): void

try {
if (path.length === 1 && path[0] === 'value') {
// Direct cell value edit
cell.value = value as string;
// Direct cell value edit — store as object if edit has an id
if (edit.id) {
(cell as any).value = { selectedEdit: edit.id, updatedAt: edit.timestamp };
cell.metadata.activeEditId = edit.id;
} else {
cell.value = value as string;
}
} else if (path.length >= 2 && path[0] === 'metadata') {
// Metadata field edit
if (path.length === 2) {
Expand Down Expand Up @@ -1036,12 +1041,20 @@ function mergeTwoCellsUsingResolverLogic(
...(theirCell.metadata?.edits || [])
].sort((a, b) => a.timestamp - b.timestamp);

// Remove duplicates based on timestamp, editMap and value, while merging validatedBy entries
// Remove duplicates: prefer edit id as primary key, fall back to composite key for legacy
const editMap = new Map<string, any>();
allEdits.forEach((edit) => {
if (edit.editMap && Array.isArray(edit.editMap)) {
const editMapKey = edit.editMap.join('.');
const key = `${edit.timestamp}:${editMapKey}:${edit.value}`;
let key: string;
if (edit.id) {
key = edit.id;
} else {
const editMapKey = edit.editMap.join('.');
const valueKey = typeof edit.value === 'object' && edit.value !== null
? JSON.stringify(edit.value)
: String(edit.value);
key = `${edit.timestamp}:${editMapKey}:${valueKey}`;
}
if (!editMap.has(key)) {
editMap.set(key, edit);
} else {
Expand All @@ -1062,6 +1075,15 @@ function mergeTwoCellsUsingResolverLogic(
}
mergedCell.metadata.edits = uniqueEdits;

// After merge, resolve activeEditId to the latest non-preview value edit
const latestValueEdit = [...uniqueEdits].reverse().find(
(e) => e.editMap && Array.isArray(e.editMap) && e.editMap.length === 1 && e.editMap[0] === 'value' && !e.preview
);
if (latestValueEdit?.id) {
mergedCell.metadata.activeEditId = latestValueEdit.id;
(mergedCell as any).value = { selectedEdit: latestValueEdit.id, updatedAt: latestValueEdit.timestamp };
}

// Merge attachments intelligently
const mergedAttachments = mergeAttachments(
ourCell.metadata?.attachments,
Expand Down
1 change: 1 addition & 0 deletions src/projectManager/utils/migrationCompletionUtils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ export const CODEX_PROJECT_MIGRATION_FLAG_KEYS = [
"globalReferencesMigrationCompleted",
"cellIdsToUuidMigrationCompleted",
"tempFilesRecoveryAndDuplicateMergeCompleted",
"editValueObjectMigrationCompleted",
] as const;

export type CodexProjectMigrationFlagKey = (typeof CODEX_PROJECT_MIGRATION_FLAG_KEYS)[number];
Expand Down
188 changes: 187 additions & 1 deletion src/projectManager/utils/migrationUtils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import { randomUUID } from "crypto";
import * as dugiteGit from "../../utils/dugiteGit";
import { CodexContentSerializer } from "@/serializer";
import { vrefData } from "@/utils/verseRefUtils/verseData";
import { EditMapUtils } from "@/utils/editMapUtils";
import { EditMapUtils, generateEditId } from "@/utils/editMapUtils";
import { EditType, CodexCellTypes } from "../../../types/enums";
import type { ValidationEntry } from "../../../types";
import { getAuthApi } from "../../extension";
Expand Down Expand Up @@ -4080,3 +4080,189 @@ export const migration_recoverTempFilesAndMergeDuplicates = async (context?: vsc
console.error("Error running temp files recovery and duplicate merge migration:", error);
}
};

/**
* Migration: Backfill edit IDs and convert cell.value to CellValueOnDisk object format.
*
* 1. Backfill `id` on all edits (cell-level and file-level) using SHA-256 deterministic hash.
* Skips edits that already have UUID-format IDs (from randomUUID).
* 2. Find matching edit for current cell.value string → set as activeEditId.
* 3. Convert cell.value from string to { selectedEdit: id, updatedAt: timestamp }.
* 4. Create INITIAL_IMPORT edit if cell has content but no matching edit.
* 5. Also backfill IDs on file-level metadata.edits.
*/
export const migration_editValueObject = async (context?: vscode.ExtensionContext) => {
try {
const workspaceFolders = vscode.workspace.workspaceFolders;
if (!workspaceFolders || workspaceFolders.length === 0) {
return;
}

const migrationKey = "editValueObjectMigrationCompleted";
const config = vscode.workspace.getConfiguration("codex-project-manager");
let hasMigrationRun = false;
try {
hasMigrationRun = config.get(migrationKey, false);
} catch (e) {
hasMigrationRun = !!context?.workspaceState.get<boolean>(migrationKey);
}
if (hasMigrationRun) {
return;
}

debug("Running edit value object migration...");

const codexFiles = await vscode.workspace.findFiles("**/*.codex");
if (codexFiles.length === 0) {
// No codex files — mark migration as done
try {
await config.update(migrationKey, true, vscode.ConfigurationTarget.Workspace);
} catch (e) {
await context?.workspaceState.update(migrationKey, true);
}
return;
}

let migratedFiles = 0;

await vscode.window.withProgress(
{
location: vscode.ProgressLocation.Notification,
title: "Migrating edit IDs and cell value objects...",
cancellable: false,
},
async (progress) => {
for (let i = 0; i < codexFiles.length; i++) {
const file = codexFiles[i];
progress.report({
message: `Processing file ${i + 1}/${codexFiles.length}`,
increment: (100 / codexFiles.length),
});

try {
const fileData = await vscode.workspace.fs.readFile(file);
const content = new TextDecoder().decode(fileData);
const notebook = JSON.parse(content);

let fileModified = false;

// Get author for deterministic IDs
let author = "anonymous";
try {
const authApi = await getAuthApi();
const userInfo = await authApi?.getUserInfo();
if (userInfo?.username) {
author = userInfo.username;
}
} catch (_) { /* ignore */ }

// Process each cell
for (const cell of (notebook.cells || [])) {
if (!cell.metadata?.edits) {
cell.metadata = cell.metadata || { id: randomUUID(), type: "text", edits: [] };
if (!cell.metadata.edits) {
cell.metadata.edits = [];
}
}

// 1. Backfill IDs on all cell-level edits
for (const edit of cell.metadata.edits) {
if (!edit.id || !isUuidFormat(edit.id)) {
const valueStr = typeof edit.value === 'object' && edit.value !== null
? JSON.stringify(edit.value)
: String(edit.value ?? '');
const authorStr = edit.author || author;
edit.id = await generateEditId(valueStr, edit.timestamp, authorStr);
fileModified = true;
}
}

// 2. Find matching edit for current cell.value
const currentValue = typeof cell.value === "string" ? cell.value : "";
if (currentValue) {
const valueEdits = cell.metadata.edits.filter(
(e: any) => e.editMap && Array.isArray(e.editMap) && e.editMap.length === 1 && e.editMap[0] === "value" && !e.preview
);

// Try to find exact match first
let matchingEdit = valueEdits.find((e: any) => e.value === currentValue);

// If no match, find latest non-preview value edit
if (!matchingEdit && valueEdits.length > 0) {
matchingEdit = valueEdits[valueEdits.length - 1];
}

// 4. Create INITIAL_IMPORT edit if cell has content but no matching value edit
if (!matchingEdit && currentValue.trim()) {
const importTimestamp = cell.metadata.edits.length > 0
? Math.min(...cell.metadata.edits.map((e: any) => e.timestamp || Date.now())) - 1000
: Date.now() - 1000;
const importId = await generateEditId(currentValue, importTimestamp, author);
const importEdit = {
id: importId,
editMap: ["value"],
value: currentValue,
timestamp: importTimestamp,
type: EditType.INITIAL_IMPORT,
author: author,
validatedBy: [],
};
cell.metadata.edits.unshift(importEdit);
matchingEdit = importEdit;
fileModified = true;
}

if (matchingEdit) {
// Set activeEditId
cell.metadata.activeEditId = matchingEdit.id;

// 3. Convert cell.value to object format
cell.value = {
selectedEdit: matchingEdit.id,
updatedAt: matchingEdit.timestamp,
};
fileModified = true;
}
}
}

// 5. Backfill IDs on file-level metadata.edits
if (notebook.metadata?.edits) {
for (const edit of notebook.metadata.edits) {
if (!edit.id || !isUuidFormat(edit.id)) {
const valueStr = typeof edit.value === 'object' && edit.value !== null
? JSON.stringify(edit.value)
: String(edit.value ?? '');
const authorStr = edit.author || author;
edit.id = await generateEditId(valueStr, edit.timestamp, authorStr);
fileModified = true;
}
}
}

if (fileModified) {
const newContent = formatJsonForNotebookFile(notebook);
await atomicWriteUriText(file, newContent);
migratedFiles++;
}
} catch (error) {
console.error(`Error migrating file ${file.fsPath}:`, error);
}
}
}
);

// Mark migration as complete
try {
await config.update(migrationKey, true, vscode.ConfigurationTarget.Workspace);
} catch (e) {
await context?.workspaceState.update(migrationKey, true);
}

if (migratedFiles > 0) {
debug(`Edit value object migration completed: ${migratedFiles} files migrated.`);
}
} catch (error) {
console.error("Error running edit value object migration:", error);
}
};
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import * as vscode from "vscode";
import { randomUUID } from "crypto";
import { CodexCellDocument } from "./codexDocument";
import { safePostMessageToPanel } from "../../utils/webviewUtils";
// Use type-only import to break circular dependency
Expand Down Expand Up @@ -2796,6 +2797,7 @@ const messageHandlers: Record<string, (ctx: MessageHandlerContext) => Promise<vo
// 1. Concatenate content and create merged edit
const mergedContent = previousContent + "<span>&nbsp;</span>" + currentContent;
const mergeEdit: EditHistory = {
id: randomUUID(),
editMap: EditMapUtils.value(),
value: mergedContent,
timestamp: timestamp + 1,
Expand Down
Loading
Loading