diff --git a/apps/roam/src/components/DiscoverSharedNodesDialog.tsx b/apps/roam/src/components/DiscoverSharedNodesDialog.tsx index fb87c6859..d5044e039 100644 --- a/apps/roam/src/components/DiscoverSharedNodesDialog.tsx +++ b/apps/roam/src/components/DiscoverSharedNodesDialog.tsx @@ -21,6 +21,7 @@ import { isFailedSharedNodeImport, type SharedNodeImportItem, } from "~/utils/importSharedNodes"; +import { importSharedRelations } from "~/utils/importSharedRelations"; import internalError from "~/utils/internalError"; import { getLoggedInClient, getSupabaseContext } from "~/utils/supabaseContext"; @@ -146,6 +147,7 @@ const DiscoverSharedNodesDialog = ({ onClose }: { onClose: () => void }) => { const [error, setError] = useState(""); const [searchTerm, setSearchTerm] = useState(""); const [selectedRids, setSelectedRids] = useState>(new Set()); + const [spaceId, setSpaceId] = useState(0); const [importProgress, setImportProgress] = useState<{ current: number; total: number; @@ -163,6 +165,7 @@ const DiscoverSharedNodesDialog = ({ onClose }: { onClose: () => void }) => { try { const context = await getSupabaseContext(); if (!context) throw new Error("Could not connect to shared persistence."); + setSpaceId(context.spaceId); const client = await getLoggedInClient(); if (!client) throw new Error("Could not connect to shared persistence."); const { sharedNodes, importedSourceRids } = await discoverSharedNodes({ @@ -242,7 +245,6 @@ const DiscoverSharedNodesDialog = ({ onClose }: { onClose: () => void }) => { sharedNodes: selectedNodes, onProgress: (current, total) => setImportProgress({ current, total }), }); - setImportResults(results); const newlyImportedRids = results .filter((item) => item.status !== "failed") .map((item) => item.sharedNode.rid); @@ -251,6 +253,8 @@ const DiscoverSharedNodesDialog = ({ onClose }: { onClose: () => void }) => { newlyImportedRids.forEach((rid) => next.add(rid)); return next; }); + await importSharedRelations(client, spaceId); + setImportResults(results); const failedImports = results.filter(isFailedSharedNodeImport); setSelectedRids( new Set(failedImports.map((item) => item.sharedNode.rid)), diff --git a/apps/roam/src/components/settings/DiscourseNodeConfigPanel.tsx b/apps/roam/src/components/settings/DiscourseNodeConfigPanel.tsx index 01a2c289d..96e2653b7 100644 --- a/apps/roam/src/components/settings/DiscourseNodeConfigPanel.tsx +++ b/apps/roam/src/components/settings/DiscourseNodeConfigPanel.tsx @@ -10,7 +10,6 @@ import { import React, { useState } from "react"; import getDiscourseNodes from "~/utils/getDiscourseNodes"; import refreshConfigTree from "~/utils/refreshConfigTree"; -import createPage from "roamjs-components/writes/createPage"; import type { CustomField } from "roamjs-components/components/ConfigPanels/types"; import posthog from "posthog-js"; import getDiscourseRelations, { @@ -18,11 +17,10 @@ import getDiscourseRelations, { } from "~/utils/getDiscourseRelations"; import { deleteBlock } from "roamjs-components/writes"; import { formatHexColor } from "./DiscourseNodeCanvasSettings"; -import setBlockProps from "~/utils/setBlockProps"; -import { DiscourseNodeSchema } from "./utils/zodSchema"; import { getGlobalSettings, setGlobalSetting } from "./utils/accessors"; import { GLOBAL_KEYS } from "./utils/settingKeys"; import { invalidateDiscourseNodeTypeCaches } from "~/utils/discourseNodeTypeCache"; +import { createDiscourseNodeSchema } from "~/utils/createDiscourseNodeSchema"; type DiscourseNodeConfigPanelProps = React.ComponentProps< CustomField["options"]["component"] @@ -82,44 +80,8 @@ const DiscourseNodeConfigPanel: React.FC = ({ className="select-none" disabled={!label} onClick={() => { - const candidateShortcut = label.slice(0, 1).toUpperCase(); - const existingShortcuts = new Set( - getDiscourseNodes() - .map((n) => n.shortcut.toUpperCase()) - .filter(Boolean), - ); - const shortcut = existingShortcuts.has(candidateShortcut) - ? "" - : candidateShortcut; - const format = `[[${label.slice(0, 3).toUpperCase()}]] - {content}`; posthog.capture("Discourse Node: Type Created", { label: label }); - void createPage({ - title: `discourse-graph/nodes/${label}`, - tree: [ - { - text: "Shortcut", - children: [{ text: shortcut }], - }, - { - text: "Tag", - children: [{ text: "" }], - }, - { - text: "Format", - children: [{ text: format }], - }, - ], - }).then((valueUid) => { - setBlockProps( - valueUid, - DiscourseNodeSchema.parse({ - text: label, - type: valueUid, - shortcut, - format, - }), - ); - invalidateDiscourseNodeTypeCaches(); + void createDiscourseNodeSchema(label).then((valueUid) => { setNodes([ ...nodes, { diff --git a/apps/roam/src/utils/createDiscourseNodeSchema.ts b/apps/roam/src/utils/createDiscourseNodeSchema.ts new file mode 100644 index 000000000..01e6cbe84 --- /dev/null +++ b/apps/roam/src/utils/createDiscourseNodeSchema.ts @@ -0,0 +1,65 @@ +import createPage from "roamjs-components/writes/createPage"; +import setBlockProps from "~/utils/setBlockProps"; +import { DiscourseNodeSchema } from "~/components/settings/utils/zodSchema"; +import { invalidateDiscourseNodeTypeCaches } from "~/utils/discourseNodeTypeCache"; +import getDiscourseNodes from "./getDiscourseNodes"; + +export const createDiscourseNodeSchema = async ( + label: string, + options?: { + shortcut?: string; + format?: string; + template?: string; + }, +): Promise => { + let { shortcut, format } = options ?? {}; + const { template } = options ?? {}; + if (shortcut === undefined) { + const candidateShortcut = label.slice(0, 1).toUpperCase(); + const existingShortcuts = new Set( + getDiscourseNodes() + .map((n) => n.shortcut.toUpperCase()) + .filter(Boolean), + ); + shortcut = existingShortcuts.has(candidateShortcut) + ? "" + : candidateShortcut; + } + format = format ?? `[[${label.slice(0, 3).toUpperCase()}]] - {content}`; + const tree = [ + { + text: "Shortcut", + children: [{ text: shortcut }], + }, + { + text: "Tag", + children: [{ text: "" }], + }, + { + text: "Format", + children: [{ text: format }], + }, + ]; + if (template != undefined) { + // TODO: Make into a tree + tree.push({ + text: "Template", + children: [{ text: template ?? "" }], + }); + } + const valueUid = await createPage({ + title: `discourse-graph/nodes/${label}`, + tree, + }); + setBlockProps( + valueUid, + DiscourseNodeSchema.parse({ + text: label, + type: valueUid, + shortcut, + format, + }), + ); + invalidateDiscourseNodeTypeCaches(); + return valueUid; +}; diff --git a/apps/roam/src/utils/createReifiedBlock.ts b/apps/roam/src/utils/createReifiedBlock.ts index ad78975f7..d39bb420f 100644 --- a/apps/roam/src/utils/createReifiedBlock.ts +++ b/apps/roam/src/utils/createReifiedBlock.ts @@ -140,7 +140,7 @@ export const createReifiedRelation = async ({ sourceUid: string; relationBlockUid: string; destinationUid: string; -}): Promise => { +}): Promise => { return await createReifiedBlock({ destinationBlockUid: await getOrCreateRelationPageUid(), schemaUid: relationBlockUid, diff --git a/apps/roam/src/utils/createRelationSchema.ts b/apps/roam/src/utils/createRelationSchema.ts new file mode 100644 index 000000000..d175bd9b5 --- /dev/null +++ b/apps/roam/src/utils/createRelationSchema.ts @@ -0,0 +1,43 @@ +import discourseConfigRef from "~/utils/discourseConfigRef"; +import createBlock from "roamjs-components/writes/createBlock"; + +export const createRelationSchema = async ({ + label, + complement, + source, + destination, +}: { + label: string; + complement: string; + source: string; + destination: string; +}) => { + const grammarNode = discourseConfigRef.tree.find( + (node) => node.text === "grammar", + ); + const relationsNode = grammarNode?.children.find( + (node) => node.text === "relations", + ); + if (!relationsNode) throw new Error("Cannot find the relation grammar"); + return await createBlock({ + parentUid: relationsNode.uid, + order: "last", + node: { + text: label, + children: [ + { + text: "source", + children: [{ text: source }], + }, + { + text: "destination", + children: [{ text: destination }], + }, + { + text: "complement", + children: [{ text: complement }], + }, + ], + }, + }); +}; diff --git a/apps/roam/src/utils/discoverSharedRelations.ts b/apps/roam/src/utils/discoverSharedRelations.ts new file mode 100644 index 000000000..cbf75bb97 --- /dev/null +++ b/apps/roam/src/utils/discoverSharedRelations.ts @@ -0,0 +1,255 @@ +import type { DGSupabaseClient } from "@repo/database/lib/client"; +import type { + CrossAppRelation, + CrossAppRelationTypeSchema, + CrossAppRelationTripleSchema, + CrossAppNodeSchema, +} from "@repo/database/crossAppContracts"; +import { + getAccountMap, + getSpaceMap, + dbRelationTripleSchemasToCrossApp, + dbRelationsToCrossApp, + dbRelationTypeSchemasToCrossApp, + dbNodeSchemasToCrossApp, +} from "@repo/database/lib/dbToCrossAppConverters"; +import { Tables } from "@repo/database/dbTypes"; +import { spaceUriAndLocalIdToRid } from "@repo/database/lib/rid"; +import { getImportedSourceRids } from "./importedSourceIdentity"; + +type Concept = Tables<"Concept">; + +export type DiscoverSharedRelationsResult = { + relations: CrossAppRelation[]; + relTripleSchemas: CrossAppRelationTripleSchema[]; + relTypeSchemas: CrossAppRelationTypeSchema[]; + nodeSchemas: CrossAppNodeSchema[]; + idToRid: Record; +}; + +export const discoverSharedRelations = async ( + client: DGSupabaseClient, + spaceId: number, + futureImportRids?: string[], +): Promise => { + const response: DiscoverSharedRelationsResult = { + relations: [], + relTripleSchemas: [], + relTypeSchemas: [], + nodeSchemas: [], + idToRid: {}, + }; + // TODO: paginate + const { data: dbAllImportableRelations, error: relError } = await client + .from("my_concepts") + .select( + "*, concepts_of_relation!inner(id, space_id, source_local_id, schema_id)", + ) + .neq("space_id", spaceId) + .eq("is_schema", false) + .gt("arity", 0); + + if (relError || !dbAllImportableRelations) { + throw relError; + } + if (dbAllImportableRelations.length === 0) return response; + const relatedNodeInfo = dbAllImportableRelations + .map((r) => r.concepts_of_relation) + .flat(); + const spaceIds = new Set(relatedNodeInfo.map(({ space_id }) => space_id!)); + const spaceMap = await getSpaceMap(client, [...spaceIds]); + const toRid = (spaceId: number, localId: string) => + spaceId in spaceMap + ? spaceUriAndLocalIdToRid(spaceMap[spaceId], localId, "note") + : undefined; + const idToRid: Record = Object.fromEntries( + relatedNodeInfo + .map( + ({ id, space_id, source_local_id }): [number, string] | undefined => { + if (id === null || space_id === null || source_local_id === null) + return; + const rid = toRid(space_id, source_local_id); + if (rid === undefined) return; + return [id, rid]; + }, + ) + .filter((x) => x !== undefined), + ); + + // We want those relations whose source/destinations are either already imported, + // or somehow connected by Rid to local nodes. + const refToLocalIds = new Set( + relatedNodeInfo + .filter(({ space_id }) => space_id === spaceId) + .map(({ id }) => id), + ); + const importedNodeRids = await getImportedSourceRids(); + if (futureImportRids !== undefined) { + futureImportRids.forEach((id) => importedNodeRids.add(id)); + } + const dbRelations = dbAllImportableRelations.filter((r) => { + const references = (r.reference_content || {}) as Record; + const sourceId = references["source"]; + const destinationId = references["destination"]; + if (!sourceId || !destinationId) return false; + return ( + (refToLocalIds.has(sourceId) || + importedNodeRids.has(idToRid[sourceId] ?? "")) && + (refToLocalIds.has(destinationId) || + importedNodeRids.has(idToRid[destinationId] ?? "")) + ); + }); + const relationSchemaIds = new Set( + dbRelations.map((r) => r.schema_id).filter((r) => r !== null), + ); + if (relationSchemaIds.size === 0) return response; + const { data: dbRelSchemas, error: relSchError } = await client + .from("my_concepts") + .select() + .in("id", [...relationSchemaIds]); + if (relSchError || !dbRelSchemas) { + throw relSchError; + } + const dbRelTripleSchemasDirect = dbRelSchemas.filter( + (r) => r.refs !== null && r.refs.length > 0, + ) as Concept[]; + const dbRelTypeSchemasDirect = dbRelSchemas.filter( + (r) => r.refs === null || r.refs.length === 0, + ) as Concept[]; + let dbRelTripleSchemas = dbRelTripleSchemasDirect; + let dbRelTypeSchemas = dbRelTypeSchemasDirect; + + const missingRelationTypeSchemaIds = new Set( + dbRelTypeSchemasDirect + .map( + (r) => + (typeof r.reference_content === "object" + ? (r.reference_content as Record) + : {})["relation_type"], + ) + .filter((id) => id !== undefined), + ); + + if (missingRelationTypeSchemaIds.size > 0) { + const { data, error: tysError } = await client + .from("my_concepts") + .select() + .in("id", [...missingRelationTypeSchemaIds]); + if (tysError || !data) { + throw tysError; + } + dbRelTypeSchemas = [...dbRelTypeSchemasDirect, ...(data as Concept[])]; + } + if (dbRelTypeSchemasDirect.length) { + // Fetch all corresponding triples and filter + const relTypeIds = dbRelTypeSchemasDirect.map((r) => r.id); + const { data, error: trsError } = await client + .from("my_concepts") + .select() + .eq("is_schema", true) + .eq("arity", 2) + .overlaps("refs", relTypeIds); + if (trsError || !data) { + throw trsError; + } + const triplesBySchemaId: Record = Object.fromEntries( + relTypeIds.map((id) => [id, []]), + ); + data.forEach((c) => { + triplesBySchemaId[ + ((c.reference_content ?? {}) as Record)["relation_type"] + ].push(c as Concept); + }); + const tripleIds = new Set(); + for (const relation of dbRelations) { + const potentialTriples = triplesBySchemaId[relation.schema_id || 0]; + if (potentialTriples === undefined) continue; + const refs = (relation.reference_content || {}) as Record; + const sourceContent = relation.concepts_of_relation.filter( + (cr) => cr.id === refs["source"], + ); + const destinationContent = relation.concepts_of_relation.filter( + (cr) => cr.id === refs["destination"], + ); + if (sourceContent.length !== 1 || destinationContent.length !== 1) + continue; + const matches = potentialTriples.filter( + (triple) => + ((triple.reference_content ?? {}) as Record)[ + "source" + ] === sourceContent[0].schema_id && + ((triple.reference_content ?? {}) as Record)[ + "destination" + ] === destinationContent[0].schema_id, + ); + if (matches.length === 1) { + const relationTripleSchemaId = matches[0].id; + tripleIds.add(relationTripleSchemaId); + // prentend that the obsidian relation referred to the triple + // for when we convert + relation.schema_id = relationTripleSchemaId; + } + } + dbRelTripleSchemas = [ + ...dbRelTripleSchemasDirect, + ...(data as Concept[]).filter((tr) => tripleIds.has(tr.id || 0)), + ]; + } + const nodeTypeSchemaIds = new Set( + dbRelTripleSchemas + .map((r) => { + const refs = (r.reference_content || {}) as Record; + return [refs.source, refs.destination]; + }) + .flat() + .filter((id) => id !== undefined), + ); + const { data: dbNodeTypeSchemas, error: nsError } = await client + .from("my_concepts") + .select() + .in("id", [...nodeTypeSchemaIds]); + if (nsError || !nodeTypeSchemaIds) { + throw nsError; + } + const authorIds = [ + ...dbRelations, + ...dbRelTripleSchemas, + ...dbRelTypeSchemas, + ...dbNodeTypeSchemas, + ] + .map((r) => r.author_id) + .filter((id) => id !== null); + const accountMap = await getAccountMap(client, [...new Set(authorIds)]); + const relTypeSchemas = await dbRelationTypeSchemasToCrossApp({ + client, + schemas: dbRelTypeSchemas, + spaceMap, + accountMap, + }); + const relTripleSchemas = await dbRelationTripleSchemasToCrossApp({ + client, + schemas: dbRelTripleSchemas, + spaceMap, + accountMap, + }); + const relations = await dbRelationsToCrossApp({ + client, + relations: dbRelations as Concept[], + accountMap, + spaceId, + spaceMap, + }); + const nodeSchemas = await dbNodeSchemasToCrossApp({ + client, + schemas: dbNodeTypeSchemas as Concept[], + spaceMap, + accountMap, + }); + return { + relations, + relTripleSchemas, + relTypeSchemas, + nodeSchemas, + idToRid, + }; +}; diff --git a/apps/roam/src/utils/importSharedRelations.ts b/apps/roam/src/utils/importSharedRelations.ts new file mode 100644 index 000000000..23ee5175d --- /dev/null +++ b/apps/roam/src/utils/importSharedRelations.ts @@ -0,0 +1,326 @@ +import type { + CrossAppRelation, + CrossAppRelationTypeSchema, + CrossAppRelationTripleSchema, + CrossAppNodeSchema, +} from "@repo/database/crossAppContracts"; +import { spaceUriAndLocalIdToRid, isRid } from "@repo/database/lib/rid"; +import { + addImportedSchemaSourceIdentity, + getImportedSourceSchemaRids, + getImportedSourceRids, + writeImportedSourceIdentity, +} from "./importedSourceIdentity"; +import getDiscourseRelations from "./getDiscourseRelations"; +import getDiscourseNodes from "./getDiscourseNodes"; +import { createDiscourseNodeSchema } from "./createDiscourseNodeSchema"; +import { createRelationSchema } from "./createRelationSchema"; +import { + createReifiedRelation, + getReifiedRelations, +} from "./createReifiedBlock"; +import { + discoverSharedRelations, + type DiscoverSharedRelationsResult, +} from "./discoverSharedRelations"; +import { DGSupabaseClient } from "@repo/database/lib/client"; + +const matchImportedNodeSchemas = async ( + nodeSchemas: CrossAppNodeSchema[], + dryRun?: boolean, +): Promise> => { + const result: Record = {}; + const nodeSchemasByRid = Object.fromEntries( + nodeSchemas.map((s) => [ + spaceUriAndLocalIdToRid(s.spaceUrl!, s.localId, "schema"), + s, + ]), + ); + const existing = await getImportedSourceSchemaRids(); + const localNodeSchemas = getDiscourseNodes(); + const localNodeSchemasByLabel = Object.fromEntries( + localNodeSchemas.map((s) => [s.text.toLowerCase(), s]), + ); + const localNodeSchemasByLocalId = Object.fromEntries( + localNodeSchemas.map((s) => [s.type, s]), + ); + + for (const [rid, schema] of Object.entries(nodeSchemasByRid)) { + let blockUid = existing[rid]; + if (blockUid !== undefined) { + result[rid] = blockUid; + continue; + } else if (schema.localId in localNodeSchemasByLocalId) { + blockUid = localNodeSchemasByLocalId[schema.localId].type; + } else if (schema.label.toLowerCase() in localNodeSchemasByLabel) { + blockUid = localNodeSchemasByLabel[schema.label.toLowerCase()].type; + } else if (dryRun !== true) { + // create a new node schema + blockUid = await createDiscourseNodeSchema(schema.label, { + template: schema.template, + // TODO: colour, other metadata? + }); + } + if (blockUid !== undefined) { + result[rid] = blockUid; + await addImportedSchemaSourceIdentity({ + blockUid, + sourceNodeRid: rid, + sourceModifiedAt: (schema.modifiedAt ?? new Date()).toISOString(), + }); + } + } + return result; +}; + +const matchImportedRelationSchemas = async ({ + nodeSchemaRidToLocalId, + relationTypeSchemas, + relationTripleSchemas, + dryRun, +}: { + nodeSchemaRidToLocalId: Record; + relationTypeSchemas: CrossAppRelationTypeSchema[]; + relationTripleSchemas: CrossAppRelationTripleSchema[]; + dryRun?: boolean; +}): Promise> => { + const result: Record = {}; + const relationSchemas = getDiscourseRelations(); + const existing = await getImportedSourceSchemaRids(); + const relationTypeSchemasBySpaceUriAndLocalId: Record< + string, + Record + > = {}; + relationTypeSchemas.forEach((s) => { + if (s.spaceUrl === undefined) return; + if (s.spaceUrl in relationTypeSchemasBySpaceUriAndLocalId) + relationTypeSchemasBySpaceUriAndLocalId[s.spaceUrl][s.localId] = s; + else + relationTypeSchemasBySpaceUriAndLocalId[s.spaceUrl] = { [s.localId]: s }; + }); + const localRelationTripleSchemasByLocalId = Object.fromEntries( + relationSchemas.map((s) => [s.id, s]), + ); + + for (const tripleSchema of relationTripleSchemas) { + const rid = spaceUriAndLocalIdToRid( + tripleSchema.spaceUrl!, + tripleSchema.localId, + "schema", + ); + let blockUid = existing[rid]; + if (blockUid !== undefined) { + result[rid] = blockUid; + continue; + } + if (tripleSchema.localId in localRelationTripleSchemasByLocalId) { + blockUid = localRelationTripleSchemasByLocalId[tripleSchema.localId].id; + } else { + const sourceTypeRid = isRid(tripleSchema.sourceType) + ? tripleSchema.sourceType + : spaceUriAndLocalIdToRid( + tripleSchema.spaceUrl!, + tripleSchema.sourceType, + "schema", + ); + const destinationTypeRid = isRid(tripleSchema.destinationType) + ? tripleSchema.destinationType + : spaceUriAndLocalIdToRid( + tripleSchema.spaceUrl!, + tripleSchema.destinationType, + "schema", + ); + const source = nodeSchemaRidToLocalId[sourceTypeRid] ?? "missing"; + const destination = + nodeSchemaRidToLocalId[destinationTypeRid] ?? "missing"; + if (!dryRun && (source === "missing" || destination === "missing")) + throw new Error("Missing source or destination"); + const relationType = + relationTypeSchemasBySpaceUriAndLocalId[tripleSchema.spaceUrl!][ + tripleSchema.relation ?? "" + ]; + + const label = tripleSchema.label ?? relationType?.label; + const complement = tripleSchema.complement ?? relationType?.complement; + const match = relationSchemas.filter( + (r) => + r.label.toLowerCase() === label.toLowerCase() && + r.source === source && + r.destination === destination, + ); + if (match.length > 1) { + throw new Error("multiple matches"); + } + if (match.length === 1) { + blockUid = match[0].id; + } else if (dryRun !== true) { + blockUid = await createRelationSchema({ + label, + complement, + source, + destination, + }); + } + } + if (blockUid !== undefined) { + result[rid] = blockUid; + await addImportedSchemaSourceIdentity({ + blockUid, + sourceNodeRid: rid, + sourceModifiedAt: (tripleSchema.modifiedAt ?? new Date()).toISOString(), + }); + } + } + return result; +}; + +const importRelations = async ( + schemaRidToLocalId: Record, + relations: CrossAppRelation[], + dryRun?: boolean, +): Promise => { + let created = 0; + const existing = await getImportedSourceRids(); + const allRelations = await getReifiedRelations(); + for (const relation of relations) { + const sourceNodeRid = spaceUriAndLocalIdToRid( + relation.spaceUrl!, + relation.localId, + "relation", + ); + if (existing.has(sourceNodeRid)) continue; + const relationBlockUid = + schemaRidToLocalId[ + spaceUriAndLocalIdToRid( + relation.spaceUrl!, + relation.relationType, + "schema", + ) + ] ?? "missing"; + if (relationBlockUid === "missing") + throw new Error(`Missing relation type: ${relation.relationType}`); + const relSource = isRid(relation.source) + ? relation.source + : spaceUriAndLocalIdToRid(relation.spaceUrl!, relation.source, "note"); + const relDestination = isRid(relation.destination) + ? relation.destination + : spaceUriAndLocalIdToRid( + relation.spaceUrl!, + relation.destination, + "note", + ); + const sourceUid = schemaRidToLocalId[relSource] ?? "missing"; + const destinationUid = schemaRidToLocalId[relDestination] ?? "missing"; + if (dryRun) { + if ( + relationBlockUid === "missing" || + sourceUid === "missing" || + destinationUid === "missing" + ) { + created++; + continue; + } + } else { + if (relationBlockUid === "missing") + throw new Error(`Missing relation type: ${relation.relationType}`); + if (sourceUid === "missing") + throw new Error(`Missing relation source: ${relation.source}`); + if (destinationUid === "missing") + throw new Error( + `Missing relation destination: ${relation.destination}`, + ); + } + const existingRel = allRelations.filter( + (r) => + r.hasSchema == relationBlockUid && + r.sourceUid == sourceUid && + r.destinationUid == r.destinationUid, + ); + if (existingRel.length > 1) throw new Error("Multiple matching relations"); + let uid: string | undefined; + if (existingRel.length === 1) { + uid = existingRel[0].relationId; + } else if (dryRun !== true) { + uid = await createReifiedRelation({ + sourceUid, + destinationUid, + relationBlockUid, + }); + created++; + } else created++; + if (uid !== undefined) { + await writeImportedSourceIdentity({ + pageUid: uid, + sourceNodeRid, + sourceModifiedAt: (relation.modifiedAt ?? new Date()).toISOString(), + }); + } + } + return created; +}; + +export const importSharedRelations = async ( + client: DGSupabaseClient, + spaceId: number, +) => { + const { relations, relTripleSchemas, relTypeSchemas, nodeSchemas, idToRid } = + await discoverSharedRelations(client, spaceId); + let ridToId = Object.fromEntries( + Object.entries(idToRid).map(([id, rid]) => [rid, id]), + ); + const nodeSchemasMap = await matchImportedNodeSchemas(nodeSchemas); + ridToId = { ...ridToId, ...nodeSchemasMap }; + const relationSchemaMap = await matchImportedRelationSchemas({ + nodeSchemaRidToLocalId: ridToId, + relationTypeSchemas: relTypeSchemas, + relationTripleSchemas: relTripleSchemas, + }); + ridToId = { ...ridToId, ...relationSchemaMap }; + await importRelations(ridToId, relations); +}; + +type CountImportSideEffect = DiscoverSharedRelationsResult & { + numNewNodeSchemas: number; + numNewRelTripleSchemas: number; + numNewRelations: number; +}; + +export const discoverSharedRelationsPreflight = async ( + client: DGSupabaseClient, + spaceId: number, + futureImportRids?: string[], +): Promise => { + const result = await discoverSharedRelations( + client, + spaceId, + futureImportRids, + ); + const { relations, relTripleSchemas, relTypeSchemas, nodeSchemas, idToRid } = + result; + let ridToId = Object.fromEntries( + Object.entries(idToRid).map(([id, rid]) => [rid, id]), + ); + const nodeSchemasMap = await matchImportedNodeSchemas(nodeSchemas, true); + const numNewNodeSchemas = + nodeSchemas.length - Object.keys(nodeSchemasMap).length; + ridToId = { ...ridToId, ...nodeSchemasMap }; + const relationSchemaMap = await matchImportedRelationSchemas({ + nodeSchemaRidToLocalId: ridToId, + relationTypeSchemas: relTypeSchemas, + relationTripleSchemas: relTripleSchemas, + dryRun: true, + }); + const numNewRelTripleSchemas = + relTripleSchemas.length - Object.keys(relationSchemaMap).length; + ridToId = { + ...ridToId, + ...relationSchemaMap, + }; + const numNewRelations = await importRelations(ridToId, relations, true); + return { + ...result, + numNewNodeSchemas, + numNewRelations, + numNewRelTripleSchemas, + }; +}; diff --git a/apps/roam/src/utils/importedSourceIdentity.ts b/apps/roam/src/utils/importedSourceIdentity.ts index 64588918a..36cbd653f 100644 --- a/apps/roam/src/utils/importedSourceIdentity.ts +++ b/apps/roam/src/utils/importedSourceIdentity.ts @@ -1,6 +1,7 @@ import type { Rid } from "@repo/database/crossAppContracts"; +import { isRid } from "@repo/database/lib/rid"; import { DISCOURSE_GRAPH_PROP_NAME } from "./createReifiedBlock"; -import getBlockProps, { type json } from "./getBlockProps"; +import getBlockProps, { normalizeProps, type json } from "./getBlockProps"; import { setBlockPropsAsync } from "./setBlockProps"; export type ImportedSourceIdentity = { @@ -9,6 +10,7 @@ export type ImportedSourceIdentity = { }; export const IMPORTED_FROM_PROP_KEY = "importedFrom"; +export const IMPORTED_FROM_SCHEMAS_PROP_KEY = "importedFromSchemas"; const SOURCE_NODE_RID_KEY = "sourceNodeRid"; const SOURCE_MODIFIED_AT_KEY = "sourceModifiedAt"; @@ -32,6 +34,27 @@ const parseImportedSourceIdentity = ( return { sourceModifiedAt, sourceNodeRid }; }; +export const parseImportedFromSchemas = ( + props: Record, +): ImportedSourceIdentity[] => { + const results: ImportedSourceIdentity[] = []; + const discourseGraphProps = props[DISCOURSE_GRAPH_PROP_NAME]; + if (!isJsonObject(discourseGraphProps)) return results; + + const importedFrom = discourseGraphProps[IMPORTED_FROM_SCHEMAS_PROP_KEY]; + if (!isJsonObject(importedFrom)) return results; + + for (const [sourceNodeRid, data] of Object.entries(importedFrom)) { + if (!isRid(sourceNodeRid)) continue; + if (!isJsonObject(data)) continue; + const sourceModifiedAt = importedFrom[SOURCE_MODIFIED_AT_KEY]; + if (typeof sourceModifiedAt !== "string") continue; + results.push({ sourceNodeRid, sourceModifiedAt }); + } + + return results; +}; + export const readImportedSourceIdentity = ( pageUid: string, ): ImportedSourceIdentity | undefined => @@ -60,6 +83,32 @@ export const writeImportedSourceIdentity = async ({ }); }; +export const addImportedSchemaSourceIdentity = async ({ + blockUid, + sourceModifiedAt, + sourceNodeRid, +}: { + blockUid: string; + sourceModifiedAt: string; + sourceNodeRid: string; +}): Promise => { + const existing = getBlockProps(blockUid)[DISCOURSE_GRAPH_PROP_NAME]; + const discourseGraphProps = isJsonObject(existing) ? existing : {}; + let importedFromSchemaData = + discourseGraphProps[IMPORTED_FROM_SCHEMAS_PROP_KEY]; + if (!isJsonObject(importedFromSchemaData)) importedFromSchemaData = {}; + importedFromSchemaData[sourceNodeRid] = { + [SOURCE_MODIFIED_AT_KEY]: sourceModifiedAt, + }; + + await setBlockPropsAsync(blockUid, { + [DISCOURSE_GRAPH_PROP_NAME]: { + ...discourseGraphProps, + [IMPORTED_FROM_SCHEMAS_PROP_KEY]: importedFromSchemaData, + }, + }); +}; + export const getImportedSourceRids = async (): Promise> => { const query = `[:find [?rid ...] :where @@ -95,3 +144,53 @@ export const findImportedNodeUidBySourceRid = async ( const [uid] = first as unknown[]; return typeof uid === "string" ? uid : null; }; + +export const findImportedSchemaUidBySourceRid = async ( + sourceNodeRid: string, +): Promise => { + const query = `[:find ?uid + :in $ ?sourceNodeRid + :where + [?page :block/uid ?uid] + [?page :block/props ?props] + [(get ?props :${DISCOURSE_GRAPH_PROP_NAME}) ?dgData] + [(get ?dgData :${IMPORTED_FROM_SCHEMAS_PROP_KEY}) ?importedFrom] + [(get ?importedFrom ?sourceNodeRid) ?ridData] + [(get ?ridData :${SOURCE_MODIFIED_AT_KEY}) ?modified]]`; + const result = (await window.roamAlphaAPI.data.async.q( + query, + sourceNodeRid, + )) as unknown[]; + + const [first] = result; + if (!Array.isArray(first)) return null; + const [uid] = first as unknown[]; + return typeof uid === "string" ? uid : null; +}; + +export const getImportedSourceSchemaRids = async (): Promise< + Record +> => { + const query = `[:find ?uid ?importedFrom + :where + [?block :block/uid ?uid] + [?block :block/props ?props] + [(get ?props :${DISCOURSE_GRAPH_PROP_NAME}) ?dgData] + [(get ?dgData :${IMPORTED_FROM_SCHEMAS_PROP_KEY}) ?importedFrom]]`; + const result = (await window.roamAlphaAPI.data.async.q(query)) as [ + string, + Record, + ][]; + + const rids: Record = {}; + for (const [uid, rawImportedFrom] of result) { + const importedFrom = normalizeProps(rawImportedFrom) as Record< + string, + json + >; + for (const sourceNodeRid of Object.keys(importedFrom)) { + if (isRid(sourceNodeRid)) rids[sourceNodeRid] = uid; + } + } + return rids; +}; diff --git a/packages/database/src/lib/dbToCrossAppConverters.ts b/packages/database/src/lib/dbToCrossAppConverters.ts new file mode 100644 index 000000000..27e359061 --- /dev/null +++ b/packages/database/src/lib/dbToCrossAppConverters.ts @@ -0,0 +1,378 @@ +import { + CrossAppNodeSchema, + CrossAppRelationTypeSchema, + CrossAppRelationTripleSchema, + CrossAppRelation, +} from "../crossAppContracts"; +import { Tables, Json } from "../dbTypes"; +import type { DGSupabaseClient } from "./client"; +import { ridToSpaceUriAndLocalId, spaceUriAndLocalIdToRid, isRid } from "./rid"; + +type Concept = Tables<"Concept">; + +const getConceptMap = async ({ + client, + conceptIds, + spaceId, + spaceMap, +}: { + client: DGSupabaseClient; + conceptIds: number[]; + spaceId?: number; + spaceMap?: Record; +}): Promise> => { + const request = await client + .from("Concept") + .select("id, space_id, source_local_id") + .in("id", conceptIds) + .not("source_local_id", "is", null); + if (request.error) throw request.error; + return Object.fromEntries( + (request.data || []).map(({ id, source_local_id, space_id }) => [ + id, + space_id === spaceId || spaceId === undefined || spaceMap === undefined + ? source_local_id + : spaceUriAndLocalIdToRid(spaceMap[space_id]!, source_local_id), + ]), + ); +}; + +export const getAccountMap = async ( + client: DGSupabaseClient, + accountIds: number[], +): Promise> => { + const request = await client + .from("PlatformAccount") + .select("id,account_local_id") + .in("id", accountIds); + if (request.error) throw request.error; + return Object.fromEntries( + (request.data || []).map(({ id, account_local_id }) => [ + id, + account_local_id, + ]), + ); +}; + +export const getSpaceMap = async ( + client: DGSupabaseClient, + spaceIds?: number[], +): Promise> => { + let query = client.from("my_spaces").select("id, url"); + if (spaceIds !== undefined) query = query.in("id", [...spaceIds]); + + const { data, error } = await query; + if (error || !data) { + throw error; + } + return Object.fromEntries(data.map(({ id, url }) => [id!, url!])); +}; + +const asSimpleLocalId = ( + rid: string | undefined, + spaceUrl: string | undefined, + optional?: boolean, +): string | undefined => { + if (rid === undefined) return undefined; + if (!isRid(rid)) return rid; + const { spaceUri, sourceLocalId } = ridToSpaceUriAndLocalId(rid); + if (spaceUrl === spaceUri) return sourceLocalId; + if (optional !== true) throw new Error("Unexpected spaceUri"); + return rid; +}; + +export const dbNodeSchemaToCrossApp = ( + schema: Concept, + spaceMap: Record, + accountMap: Record, +): CrossAppNodeSchema => { + const { template, template_content, ...other } = + schema.literal_content as Record; + const authorId = accountMap[schema.author_id || 0]; + if (authorId === undefined) throw new Error("Missing author"); + return { + spaceUrl: spaceMap[schema.space_id], + localId: schema.source_local_id!, + createdAt: new Date(schema.created + "Z"), + modifiedAt: new Date(schema.last_modified + "Z"), + label: schema.name, + metadata: other, + template: template_content as string | undefined, + templateTitle: template as string | undefined, + authorId, + }; +}; + +export const dbNodeSchemasToCrossApp = async ({ + client, + schemas, + spaceMap, + accountMap, +}: { + client: DGSupabaseClient; + schemas: Concept[]; + spaceMap?: Record; + accountMap?: Record; +}): Promise => { + if (spaceMap === undefined) spaceMap = await getSpaceMap(client); + if (accountMap === undefined) { + const authorIds = new Set( + schemas.map((r) => r.author_id).filter((id) => typeof id === "number"), + ); + accountMap = await getAccountMap(client, [...authorIds]); + } + return schemas.map((r) => dbNodeSchemaToCrossApp(r, spaceMap, accountMap)); +}; + +export const dbRelationTypeSchemaToCrossApp = ( + schema: Concept, + spaceMap: Record, + accountMap: Record, +): CrossAppRelationTypeSchema => { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const { roles, label, complement, ...other } = + schema.literal_content as Record; + const authorId = accountMap[schema.author_id || 0]; + if (authorId === undefined) throw new Error("Missing author"); + return { + spaceUrl: spaceMap[schema.space_id], + localId: schema.source_local_id!, + createdAt: new Date(schema.created + "Z"), + modifiedAt: new Date(schema.last_modified + "Z"), + metadata: other, + label: label as string, + complement: complement as string, + authorId, + }; +}; + +export const dbRelationTypeSchemasToCrossApp = async ({ + client, + schemas, + spaceMap, + accountMap, +}: { + client: DGSupabaseClient; + schemas: Concept[]; + spaceMap?: Record; + accountMap?: Record; +}): Promise => { + if (spaceMap === undefined) spaceMap = await getSpaceMap(client); + if (accountMap === undefined) { + const authorIds = new Set( + schemas.map((r) => r.author_id).filter((id) => typeof id === "number"), + ); + accountMap = await getAccountMap(client, [...authorIds]); + } + + return schemas.map((r) => + dbRelationTypeSchemaToCrossApp(r, spaceMap, accountMap), + ); +}; + +export const dbRelationTripleSchemaToCrossApp = ({ + schema, + spaceMap, + accountMap, + conceptMap, +}: { + schema: Concept; + spaceMap: Record; + accountMap: Record; + conceptMap: Record; +}): CrossAppRelationTripleSchema => { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const { roles, label, complement, ...other } = + schema.literal_content as Record; + const authorId = accountMap[schema.author_id || 0]; + if (authorId === undefined) throw new Error("Missing author"); + const references = (schema.reference_content ?? {}) as Record; + const spaceUrl = spaceMap[schema.space_id]; + const relation = asSimpleLocalId( + conceptMap[references["relation_type"] ?? 0], + spaceUrl, + ); + const sourceType = asSimpleLocalId( + conceptMap[references["source"] ?? 0], + spaceUrl, + ); + const destinationType = asSimpleLocalId( + conceptMap[references["source"] ?? 0], + spaceUrl, + ); + if (sourceType === undefined) throw new Error("Missing source type"); + if (destinationType === undefined) + throw new Error("Missing destination type"); + const base = { + spaceUrl, + localId: schema.source_local_id!, + createdAt: new Date(schema.created + "Z"), + modifiedAt: new Date(schema.last_modified + "Z"), + metadata: other, + authorId, + sourceType, + destinationType, + }; + if (relation) { + return { + ...base, + relation, + }; + } else { + if (typeof label !== "string" || typeof complement !== "string") + throw new Error("Missing either relation_type or relation_type data"); + return { + ...base, + label, + complement, + }; + } +}; + +export const dbRelationTripleSchemasToCrossApp = async ({ + client, + schemas, + spaceMap, + accountMap, + conceptMap, +}: { + client: DGSupabaseClient; + schemas: Concept[]; + spaceMap?: Record; + accountMap?: Record; + conceptMap?: Record; +}): Promise => { + if (spaceMap === undefined) spaceMap = await getSpaceMap(client); + if (accountMap === undefined) { + const authorIds = new Set( + schemas.map((r) => r.author_id).filter((id) => typeof id === "number"), + ); + accountMap = await getAccountMap(client, [...authorIds]); + } + if (conceptMap === undefined) { + const schemaIds = schemas + .map((r) => { + const refs = (r.reference_content ?? {}) as Record< + string, + number | number[] + >; + return [ + refs["source"] ?? [], + refs["destination"] ?? [], + refs["relation_type"] ?? [], + ]; + }) + .flat(2); + conceptMap = await getConceptMap({ + client, + conceptIds: [...new Set(schemaIds)], + }); + } + + return schemas.map((schema) => + dbRelationTripleSchemaToCrossApp({ + schema, + spaceMap, + accountMap, + conceptMap, + }), + ); +}; + +export const dbRelationToCrossApp = ({ + relation, + spaceMap, + accountMap, + conceptMap, +}: { + relation: Concept; + spaceMap: Record; + accountMap: Record; + conceptMap: Record; +}): CrossAppRelation => { + const authorId = accountMap[relation.author_id || 0]; + if (authorId === undefined) throw new Error("Missing author"); + const references = (relation.reference_content ?? {}) as Record< + string, + number + >; + const spaceUrl = spaceMap[relation.space_id]; + const relationType = asSimpleLocalId( + conceptMap[relation.schema_id || 0], + spaceUrl, + ); + if (relationType === undefined) throw new Error("Missing relationType"); + const source = asSimpleLocalId( + conceptMap[references["source"] || 0], + spaceUrl, + true, + ); + if (source === undefined) throw new Error("Missing source"); + const destination = asSimpleLocalId( + conceptMap[references["destination"] || 0], + spaceUrl, + true, + ); + if (destination === undefined) throw new Error("Missing destination"); + + return { + spaceUrl, + localId: relation.source_local_id!, + authorId, + createdAt: new Date(relation.created + "Z"), + modifiedAt: new Date(relation.last_modified + "Z"), + source, + destination, + relationType, + }; +}; + +export const dbRelationsToCrossApp = async ({ + client, + relations, + spaceId, + accountMap, + conceptMap, + spaceMap, +}: { + client: DGSupabaseClient; + relations: Concept[]; + spaceId: number; + accountMap?: Record; + conceptMap?: Record; + spaceMap?: Record; +}): Promise => { + if (accountMap === undefined) { + const authorIds = new Set( + relations.map((r) => r.author_id).filter((id) => typeof id === "number"), + ); + accountMap = await getAccountMap(client, [...authorIds]); + } + if (spaceMap === undefined) { + const spaceIds = relations.map((r) => r.space_id); + spaceMap = await getSpaceMap(client, [...new Set(spaceIds)]); + } + if (conceptMap === undefined) { + const nodeIds = relations + .map((r) => { + const refs = (r.reference_content ?? {}) as Record< + string, + number | number[] + >; + return [refs["source"] ?? [], refs["destination"] ?? []]; + }) + .flat(2); + const schemaIds = relations + .map((r) => r.schema_id) + .filter((id) => id !== null); + conceptMap = await getConceptMap({ + client, + conceptIds: [...new Set([...schemaIds, ...nodeIds])], + spaceId, + spaceMap, + }); + } + return relations.map((relation) => + dbRelationToCrossApp({ relation, spaceMap, accountMap, conceptMap }), + ); +};