diff --git a/helm/charts/api/templates/_helpers.tpl b/helm/charts/api/templates/_helpers.tpl index acec7a5..a3ddb85 100644 --- a/helm/charts/api/templates/_helpers.tpl +++ b/helm/charts/api/templates/_helpers.tpl @@ -63,9 +63,7 @@ Returns the tag of the chart. Returns the cloud provider name. */}} {{- define "api.cloudProviderFlavor" -}} -{{- if .Values.global.cloudProvider.flavor }} - {{- .Values.global.cloudProvider.flavor -}} -{{- else if .Values.cloudProvider -}} +{{- if .Values.cloudProvider -}} {{- .Values.cloudProvider.flavor | default "minikube" -}} {{- else -}} {{ "minikube" }} @@ -76,9 +74,7 @@ Returns the cloud provider name. Returns the cloud provider docker registry url. */}} {{- define "api.cloudProviderDockerRegistryUrl" -}} -{{- if .Values.global.cloudProvider.dockerRegistryUrl }} - {{- printf "%s/" .Values.global.cloudProvider.dockerRegistryUrl -}} -{{- else if .Values.cloudProvider.dockerRegistryUrl -}} +{{- if .Values.cloudProvider.dockerRegistryUrl -}} {{- printf "%s/" .Values.cloudProvider.dockerRegistryUrl -}} {{- else -}} {{- end -}} @@ -88,9 +84,7 @@ Returns the cloud provider docker registry url. Returns the cloud provider image pull secret name. */}} {{- define "api.cloudProviderImagePullSecretName" -}} -{{- if .Values.global.cloudProvider.imagePullSecretName }} - {{- .Values.global.cloudProvider.imagePullSecretName -}} -{{- else if .Values.cloudProvider.imagePullSecretName -}} +{{- if .Values.cloudProvider.imagePullSecretName -}} {{- .Values.cloudProvider.imagePullSecretName -}} {{- end -}} {{- end -}} diff --git a/helm/charts/synchronizer/templates/_helpers.tpl b/helm/charts/synchronizer/templates/_helpers.tpl index 18d4645..9905225 100644 --- a/helm/charts/synchronizer/templates/_helpers.tpl +++ b/helm/charts/synchronizer/templates/_helpers.tpl @@ -40,9 +40,7 @@ app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} {{- define "synchronizer.cloudProviderFlavor" -}} -{{- if .Values.global.cloudProvider.flavor }} - {{- .Values.global.cloudProvider.flavor -}} -{{- else if .Values.cloudProvider -}} +{{- if .Values.cloudProvider -}} {{- .Values.cloudProvider.flavor | default "minikube" -}} {{- else -}} {{ "minikube" }} @@ -50,18 +48,14 @@ app.kubernetes.io/instance: {{ .Release.Name }} {{- end -}} {{- define "synchronizer.cloudProviderDockerRegistryUrl" -}} -{{- if .Values.global.cloudProvider.dockerRegistryUrl }} - {{- printf "%s/" .Values.global.cloudProvider.dockerRegistryUrl -}} -{{- else if .Values.cloudProvider.dockerRegistryUrl -}} +{{- if .Values.cloudProvider.dockerRegistryUrl -}} {{- printf "%s/" .Values.cloudProvider.dockerRegistryUrl -}} {{- else -}} {{- end -}} {{- end -}} {{- define "synchronizer.cloudProviderImagePullSecretName" -}} -{{- if .Values.global.cloudProvider.imagePullSecretName }} - {{- .Values.global.cloudProvider.imagePullSecretName -}} -{{- else if .Values.cloudProvider.imagePullSecretName -}} +{{- if .Values.cloudProvider.imagePullSecretName -}} {{- .Values.cloudProvider.imagePullSecretName -}} {{- end -}} {{- end -}} diff --git a/helm/charts/synchronizer/templates/configmap.yaml b/helm/charts/synchronizer/templates/configmap.yaml index 35cca82..66a4cc0 100644 --- a/helm/charts/synchronizer/templates/configmap.yaml +++ b/helm/charts/synchronizer/templates/configmap.yaml @@ -36,6 +36,10 @@ data: ALIASES_FILE_PATH: {{ .Values.env.aliasesFile | quote }} + {{- if .Values.typeMap.enabled }} + TYPE_MAP_FILE_PATH: {{ .Values.env.typeMapFile | quote }} + {{- end }} + {{- with .Values.dbs.s3 }} S3_ENDPOINT: {{ .endpoint | quote }} S3_REGION: {{ .region | quote }} @@ -54,6 +58,7 @@ data: ENRICHMENT_API_URL: {{ .api | quote }} ENRICHMENT_PROPERTIES_PATH: {{ .propertiesPath | quote }} ENRICHMENT_ALIAS_FIELD: {{ .aliasField | quote }} + ENRICHMENT_REQUEST_TIMEOUT_MILLISECONDS: {{ .requestTimeoutMilliseconds | quote }} {{- end }} {{- end }} {{- include "synchronizer.dbEnvBlock" (dict "prefix" "SOURCE_DB" "db" .Values.dbs.sourceDb) | nindent 2 }} diff --git a/helm/charts/synchronizer/templates/deployment.yaml b/helm/charts/synchronizer/templates/deployment.yaml index ec31906..c0a319f 100644 --- a/helm/charts/synchronizer/templates/deployment.yaml +++ b/helm/charts/synchronizer/templates/deployment.yaml @@ -96,6 +96,10 @@ spec: mountPath: {{ .Values.env.layersFile | dir | quote }} - name: aliases-config mountPath: {{ .Values.env.aliasesFile | dir | quote }} + {{- if .Values.typeMap.enabled }} + - name: typemap-config + mountPath: {{ .Values.env.typeMapFile | dir | quote }} + {{- end }} - name: s3-downloads mountPath: /usr/src/app/packages/synchronizer/dist/src/common/s3/downloads {{- if .Values.dbs.sourceDb.sslAuth.enabled }} @@ -120,6 +124,11 @@ spec: - name: aliases-config configMap: name: {{ include "synchronizer.fullname" . }}-aliases + {{- if .Values.typeMap.enabled }} + - name: typemap-config + configMap: + name: {{ include "synchronizer.fullname" . }}-typemap + {{- end }} - name: s3-downloads emptyDir: {} {{- if .Values.dbs.sourceDb.sslAuth.enabled }} diff --git a/helm/charts/synchronizer/templates/typemap-configmap.yaml b/helm/charts/synchronizer/templates/typemap-configmap.yaml new file mode 100644 index 0000000..673f9cd --- /dev/null +++ b/helm/charts/synchronizer/templates/typemap-configmap.yaml @@ -0,0 +1,10 @@ +{{- if and .Values.enabled .Values.typeMap.enabled -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "synchronizer.fullname" . }}-typemap + labels: + {{- include "synchronizer.labels" . | nindent 4 }} +data: + {{ .Values.env.typeMapFile | base }}: {{ .Values.typeMap.data | toPrettyJson | quote }} +{{- end }} diff --git a/helm/charts/synchronizer/values.yaml b/helm/charts/synchronizer/values.yaml index 4c7e3bb..a9912ee 100644 --- a/helm/charts/synchronizer/values.yaml +++ b/helm/charts/synchronizer/values.yaml @@ -49,6 +49,12 @@ layers: aliases: {} +typeMap: + enabled: false + data: + types: {} + geometrySubTypes: {} + env: port: 80 targetPort: 8080 @@ -66,11 +72,13 @@ env: indexNameFormat: '{layerName}_{column}_idx' layersFile: /usr/src/app/packages/synchronizer/dist/config/layers/layers.json aliasesFile: /usr/src/app/packages/synchronizer/dist/config/aliases/aliases.json + typeMapFile: /usr/src/app/packages/synchronizer/dist/config/typeMap/typeMap.json enrichment: enabled: false api: "" propertiesPath: "" aliasField: "" + requestTimeoutMilliseconds: 5000 dbs: s3: diff --git a/helm/values.yaml b/helm/values.yaml index 731f80e..51b1cad 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -17,7 +17,7 @@ mclabels: &mclabels path: /metrics cloudProvider: - flavor: openshift + flavor: &cloudProviderFlavor openshift dockerRegistryUrl: &dockerRegistryUrl '' imagePullSecretName: &pullSecret '' @@ -44,13 +44,14 @@ api: initialDelaySeconds: 60 cloudProvider: + flavor: *cloudProviderFlavor dockerRegistryUrl: *dockerRegistryUrl imagePullSecretName: *pullSecret image: repository: standard-api pullPolicy: Always - tag: "v1.0.0" + tag: "v1.1.0" startupProbe: enabled: true @@ -153,13 +154,14 @@ synchronizer: path: /liveness cloudProvider: + flavor: *cloudProviderFlavor dockerRegistryUrl: *dockerRegistryUrl imagePullSecretName: *pullSecret image: repository: standard-synchronizer pullPolicy: Always - tag: "v1.0.0" + tag: "v1.2.0" layers: - layerName: buildings_polygon @@ -167,7 +169,13 @@ synchronizer: aliases: "*": - number_of_floors: "number_of_floors" + num_floors: "number_of_floors" + + typeMap: + enabled: false + data: + types: {} + geometrySubTypes: {} env: port: 80 @@ -186,11 +194,13 @@ synchronizer: indexNameFormat: '{layerName}_{column}_idx' layersFile: /usr/src/app/packages/synchronizer/dist/config/layers/layers.json aliasesFile: /usr/src/app/packages/synchronizer/dist/config/aliases/aliases.json + typeMapFile: /usr/src/app/packages/synchronizer/dist/config/typeMap.json enrichment: enabled: false api: "" propertiesPath: "" aliasField: "" + requestTimeoutMilliseconds: 5000 dbs: s3: diff --git a/package-lock.json b/package-lock.json index 2d21f58..ad0f0db 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "packages/*" ], "dependencies": { - "@map-colonies/schemas": "1.25.1", + "@map-colonies/schemas": "https://ghatmpstorage.blob.core.windows.net/npm-packages/schemas-7a16442d5a02834556d556274900dbe792311075.tgz", "lerna": "^6.6.2" }, "devDependencies": { @@ -3707,8 +3707,8 @@ }, "node_modules/@map-colonies/schemas": { "version": "1.25.1", - "resolved": "https://registry.npmjs.org/@map-colonies/schemas/-/schemas-1.25.1.tgz", - "integrity": "sha512-diGiVQGBANxnrD0hc0if90IStpySf31o0UwoAty9yHZYDNUhTKZcgUJngcBttH73Ue2mQ3dbCMbeyZAwwY6O3A==", + "resolved": "https://ghatmpstorage.blob.core.windows.net/npm-packages/schemas-7a16442d5a02834556d556274900dbe792311075.tgz", + "integrity": "sha512-Np/DJMS6kiDWBvMTKzsF3F7EmrwRQ5QynPU4IdpFOoZAaPrh7VsfpLE/Vc6Em9UBqQA078L7XyiOCzteBJJW6w==", "license": "MIT", "peer": true }, diff --git a/package.json b/package.json index 80771d6..541b9df 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,7 @@ "typescript": "^5.9.3" }, "dependencies": { - "@map-colonies/schemas": "1.25.1", + "@map-colonies/schemas": "https://ghatmpstorage.blob.core.windows.net/npm-packages/schemas-4b2c908bb50bda87cd8081773704a9718de4c2e6.tgz", "lerna": "^6.6.2" } } diff --git a/packages/synchronizer/config/default.json b/packages/synchronizer/config/default.json index 1a9a5cb..9067f03 100644 --- a/packages/synchronizer/config/default.json +++ b/packages/synchronizer/config/default.json @@ -5,6 +5,7 @@ }, "layersFile": "./config/layers.json", "aliasesFile": "./config/aliases.json", + "typeMapFile": "./config/typeMap.json", "schedule": "*/1 * * * *", "indexNameFormat": "{layerName}_{column}_idx", "enrichment": { diff --git a/packages/synchronizer/config/layers.json b/packages/synchronizer/config/layers.json index 3688b7a..23709ec 100644 --- a/packages/synchronizer/config/layers.json +++ b/packages/synchronizer/config/layers.json @@ -1,6 +1,7 @@ [ { "layerName": "buildings_polygon", + "excludeProperties": ["id"], "enums": ["code", "attribute"] } ] diff --git a/packages/synchronizer/config/test.json b/packages/synchronizer/config/test.json index 15f3ba5..1884e75 100644 --- a/packages/synchronizer/config/test.json +++ b/packages/synchronizer/config/test.json @@ -11,6 +11,7 @@ "enabled": true, "api": "https://example.com/{layerName}", "propertiesPath": "fields_list", - "aliasField": "display_name" + "aliasField": "display_name", + "requestTimeoutMilliseconds": 10000 } } diff --git a/packages/synchronizer/config/typeMap.json b/packages/synchronizer/config/typeMap.json new file mode 100644 index 0000000..a9d13bf --- /dev/null +++ b/packages/synchronizer/config/typeMap.json @@ -0,0 +1,114 @@ +{ + "types": { + "bigint": "xsd:long", + "int8": "xsd:long", + "bigserial": "xsd:long", + "serial8": "xsd:long", + "integer": "xsd:long", + "int": "xsd:long", + "int4": "xsd:long", + "serial": "xsd:long", + "serial4": "xsd:long", + "smallint": "xsd:long", + "int2": "xsd:long", + "smallserial": "xsd:long", + "serial2": "xsd:long", + "oid": "xsd:long", + + "real": "xsd:double", + "float4": "xsd:double", + "float8": "xsd:double", + "float": "xsd:double", + "double precision": "xsd:double", + "numeric": "xsd:double", + "decimal": "xsd:double", + + "boolean": "xsd:boolean", + "bool": "xsd:boolean", + + "text": "xsd:string", + "character varying": "xsd:string", + "varchar": "xsd:string", + "character": "xsd:string", + "char": "xsd:string", + "bpchar": "xsd:string", + "name": "xsd:string", + "citext": "xsd:string", + "uuid": "xsd:string", + "json": "xsd:string", + "jsonb": "xsd:string", + "xml": "xsd:string", + "bytea": "xsd:string", + "money": "xsd:string", + "inet": "xsd:string", + "cidr": "xsd:string", + "macaddr": "xsd:string", + "macaddr8": "xsd:string", + "interval": "xsd:string", + "time": "xsd:string", + "time without time zone": "xsd:string", + "time with time zone": "xsd:string", + "timetz": "xsd:string", + "tsvector": "xsd:string", + "tsquery": "xsd:string", + "text[]": "xsd:string", + "character varying[]": "xsd:string", + "varchar[]": "xsd:string", + "integer[]": "xsd:string", + "bigint[]": "xsd:string", + "numeric[]": "xsd:string", + "boolean[]": "xsd:string", + "uuid[]": "xsd:string", + "jsonb[]": "xsd:string", + + "timestamp": "xsd:dateTime", + "timestamp with time zone": "xsd:dateTime", + "timestamp without time zone": "xsd:dateTime", + "timestamptz": "xsd:dateTime", + "date": "xsd:dateTime", + + "geometry": "gml:GeometryPropertyType", + "geography": "gml:GeometryPropertyType" + }, + "geometrySubTypes": { + "point": "gml:PointPropertyType", + "pointz": "gml:PointPropertyType", + "pointm": "gml:PointPropertyType", + "pointzm": "gml:PointPropertyType", + + "linestring": "gml:LineStringPropertyType", + "linestringz": "gml:LineStringPropertyType", + "linestringm": "gml:LineStringPropertyType", + "linestringzm": "gml:LineStringPropertyType", + + "polygon": "gml:PolygonPropertyType", + "polygonz": "gml:PolygonPropertyType", + "polygonm": "gml:PolygonPropertyType", + "polygonzm": "gml:PolygonPropertyType", + + "multipoint": "gml:MultiPointPropertyType", + "multipointz": "gml:MultiPointPropertyType", + "multipointm": "gml:MultiPointPropertyType", + "multipointzm": "gml:MultiPointPropertyType", + + "multilinestring": "gml:MultiLineStringPropertyType", + "multilinestringz": "gml:MultiLineStringPropertyType", + "multilinestringm": "gml:MultiLineStringPropertyType", + "multilinestringzm": "gml:MultiLineStringPropertyType", + + "multipolygon": "gml:MultiPolygonPropertyType", + "multipolygonz": "gml:MultiPolygonPropertyType", + "multipolygonm": "gml:MultiPolygonPropertyType", + "multipolygonzm": "gml:MultiPolygonPropertyType", + + "geometrycollection": "gml:GeometryPropertyType", + "circularstring": "gml:GeometryPropertyType", + "compoundcurve": "gml:GeometryPropertyType", + "curvepolygon": "gml:GeometryPropertyType", + "multicurve": "gml:GeometryPropertyType", + "multisurface": "gml:GeometryPropertyType", + "polyhedralsurface": "gml:GeometryPropertyType", + "tin": "gml:GeometryPropertyType", + "triangle": "gml:GeometryPropertyType" + } +} diff --git a/packages/synchronizer/src/common/config.ts b/packages/synchronizer/src/common/config.ts index 165f24e..b472e6d 100644 --- a/packages/synchronizer/src/common/config.ts +++ b/packages/synchronizer/src/common/config.ts @@ -1,8 +1,8 @@ import { type ConfigInstance, config } from '@map-colonies/config'; -import { vectorVectorStandardSynchronizerV2, type vectorVectorStandardSynchronizerV2Type } from '@map-colonies/schemas'; +import { vectorVectorStandardSynchronizerV3, type vectorVectorStandardSynchronizerV3Type } from '@map-colonies/schemas'; // Choose here the type of the config instance and import this type from the entire application -type ConfigType = ConfigInstance; +type ConfigType = ConfigInstance; let configInstance: ConfigType | undefined; @@ -13,7 +13,7 @@ let configInstance: ConfigType | undefined; */ async function initConfig(offlineMode?: boolean): Promise { configInstance = await config({ - schema: vectorVectorStandardSynchronizerV2, + schema: vectorVectorStandardSynchronizerV3, offlineMode, }); } diff --git a/packages/synchronizer/src/common/interfaces.ts b/packages/synchronizer/src/common/interfaces.ts index 25b8b09..9dd65fd 100644 --- a/packages/synchronizer/src/common/interfaces.ts +++ b/packages/synchronizer/src/common/interfaces.ts @@ -6,6 +6,7 @@ export type InsertPropertyDTO = Omit['enrichment']; diff --git a/packages/synchronizer/src/common/tracing/sync.ts b/packages/synchronizer/src/common/tracing/sync.ts index 9d3258b..0458768 100644 --- a/packages/synchronizer/src/common/tracing/sync.ts +++ b/packages/synchronizer/src/common/tracing/sync.ts @@ -20,6 +20,10 @@ export const SyncAttributes = { PROPERTIES_AFFECTED: 'properties.affected', ENUMS_AFFECTED: 'enums.affected', ALIASES_COUNT: 'aliases.count', + ENRICHMENT_URL: 'enrichment.url', + ENRICHMENT_TIMEOUT_MILLISECONDS: 'enrichment.timeout_ms', + ENRICHMENT_STATUS_CODE: 'enrichment.status_code', + ENRICHMENT_ERROR_CODE: 'enrichment.error_code', } as const; export type SyncAttributes = (typeof SyncAttributes)[keyof typeof SyncAttributes]; diff --git a/packages/synchronizer/src/sync/SyncModel.ts b/packages/synchronizer/src/sync/SyncModel.ts index 5add8a9..7de50f4 100644 --- a/packages/synchronizer/src/sync/SyncModel.ts +++ b/packages/synchronizer/src/sync/SyncModel.ts @@ -12,7 +12,7 @@ import { ConfigType } from '@common/config'; import { InsertPropertyDTO, LayerEnums } from '@src/common/interfaces'; import { S3Repository } from '@src/common/s3/s3Repository'; import { CaseInsensitiveMap } from '@src/common/caseInsensitiveMap'; -import { columnInfosToProperties, schemaOf } from './helpers'; +import { columnInfosToProperties, excludedPropertySet, schemaOf } from './helpers'; import { EnumDistinctValuesError, EnumIndexError, @@ -25,7 +25,8 @@ import { TableColumnsQueryError, } from './errors'; import { fetchPropertyAliases } from './aliasEnricher'; -import { FileReader } from './fileReader'; +import type { FileAliases } from './fileReader'; +import type { TypeMap } from './typeMap'; import { LuaLayer, parseLuaLayers } from './luaParser'; export interface ColumnInfo { @@ -42,8 +43,7 @@ export class SyncModel { @inject(LAYER_REPOSITORY_SYMBOL) private readonly layerRepository: Repository, @inject(PROPERTY_REPOSITORY_SYMBOL) private readonly propertyRepository: Repository, @inject(ENUMS_REPOSITORY_SYMBOL) private readonly enumsRepository: Repository, - private readonly luaRepository: S3Repository, - private readonly fileReader: FileReader + private readonly luaRepository: S3Repository ) {} public async syncLayer(layerName: string, layerId: number, alias?: string): Promise { @@ -146,13 +146,19 @@ export class SyncModel { }); } - public async syncProperties(layer: LayerEnums, layerId: number): Promise { + public async syncProperties(layer: LayerEnums, layerId: number, typeMap: TypeMap, fileAliases: FileAliases): Promise { return startActivePromisifiedSpan( SyncSpanName.SYNC_PROPERTIES, { [SyncAttributes.LAYER_NAME]: layer.layerName, [SyncAttributes.LAYER_ID]: layerId }, contextAPI.active(), async (span) => { - const properties = columnInfosToProperties(await this.getTableColumns(layer.layerName), layer.layerName, (columnName, udtName) => { + const excluded = excludedPropertySet(layer.excludeProperties); + const columns = (await this.getTableColumns(layer.layerName)).filter(({ columnName }) => !excluded.has(columnName.toLowerCase())); + if (excluded.size > 0) { + this.logger.debug({ layerName: layer.layerName, excludeProperties: layer.excludeProperties }, 'Excluding properties from sync'); + } + + const properties = columnInfosToProperties(columns, layer.layerName, typeMap, (columnName, udtName) => { this.logger.warn({ layerName: layer.layerName, columnName, udtName }, 'Unknown column type, skipping property'); }); @@ -168,9 +174,8 @@ export class SyncModel { ); } } - const allFileAliases = await this.fileReader.readAliases(this.config.get('aliasesFile')); - const globalAliases = allFileAliases.get('*') ?? new Map(); - const layerAliases = allFileAliases.get(layer.layerName) ?? new Map(); + const globalAliases = fileAliases.get('*') ?? new Map(); + const layerAliases = fileAliases.get(layer.layerName) ?? new Map(); const aliases = new CaseInsensitiveMap([...apiAliases, ...globalAliases, ...layerAliases]); for (const property of properties) { diff --git a/packages/synchronizer/src/sync/aliasEnricher.ts b/packages/synchronizer/src/sync/aliasEnricher.ts index 2dd10b1..9b078a6 100644 --- a/packages/synchronizer/src/sync/aliasEnricher.ts +++ b/packages/synchronizer/src/sync/aliasEnricher.ts @@ -1,19 +1,16 @@ import axios from 'axios'; -import type { JsonObject, JsonValue } from 'type-fest'; +import type { JsonObject } from 'type-fest'; import { context as contextAPI } from '@opentelemetry/api'; import { startActivePromisifiedSpan } from '@common/tracing/util'; import { SyncAttributes, SyncSpanName } from '@common/tracing/sync'; import type { EnrichmentConfig } from '@src/common/interfaces'; import { CaseInsensitiveMap } from '@src/common/caseInsensitiveMap'; +import { isJsonObject } from './helpers'; const resolveTemplate = (template: string, vars: Record): string => { return template.replace(/\{(\w+)\}/g, (_, key: string) => vars[key] ?? `{${key}}`); }; -export const isJsonObject = (value: JsonValue | undefined): value is JsonObject => { - return typeof value === 'object' && value !== null && !Array.isArray(value); -}; - export const resolvePath = (data: JsonObject, path: string): JsonObject | undefined => { return path.split('.').reduce((acc, key) => { const val = acc?.[key]; @@ -32,7 +29,27 @@ export const fetchPropertyAliases = async ( contextAPI.active(), async (span) => { const url = resolveTemplate(config.api, { layerName, layerId: String(layerId) }); - const { data } = await axios.get(url); + span.setAttribute(SyncAttributes.ENRICHMENT_URL, url); + span.setAttribute(SyncAttributes.ENRICHMENT_TIMEOUT_MILLISECONDS, config.requestTimeoutMilliseconds); + + let data: JsonObject; + try { + const response = await axios.get(url, { + timeout: config.requestTimeoutMilliseconds, + signal: AbortSignal.timeout(config.requestTimeoutMilliseconds), + }); + span.setAttribute(SyncAttributes.ENRICHMENT_STATUS_CODE, response.status); + data = response.data; + } catch (err) { + if (axios.isAxiosError(err)) { + span.setAttribute(SyncAttributes.ENRICHMENT_ERROR_CODE, err.code ?? 'unknown'); + if (err.response !== undefined) { + span.setAttribute(SyncAttributes.ENRICHMENT_STATUS_CODE, err.response.status); + } + } + throw err; + } + const propertiesMap = resolvePath(data, resolveTemplate(config.propertiesPath, { layerName })); const result = new CaseInsensitiveMap(); diff --git a/packages/synchronizer/src/sync/cron.ts b/packages/synchronizer/src/sync/cron.ts index 4622df2..a601e97 100644 --- a/packages/synchronizer/src/sync/cron.ts +++ b/packages/synchronizer/src/sync/cron.ts @@ -37,6 +37,9 @@ export class CronManager { span.setAttribute(SyncAttributes.LAYERS_COUNT, layers.length); span.setAttribute(SyncAttributes.LAYERS_CHANGED, layersChanged); + const typeMap = await this.fileReader.readTypeMap(this.config.get('typeMapFile')); + const fileAliases = await this.fileReader.readAliases(this.config.get('aliasesFile')); + const luaLayersMap = await this.dal.luaFileData(); for (const layer of layers) { const luaLayer = luaLayersMap.get(layer.layerName); @@ -46,7 +49,7 @@ export class CronManager { } await this.dal.syncLayer(layer.layerName, luaLayer.layerId, luaLayer.alias); try { - const affected = await this.dal.syncProperties(layer, luaLayer.layerId); + const affected = await this.dal.syncProperties(layer, luaLayer.layerId, typeMap, fileAliases); this.logger.info({ layer, affected }, 'synced properties'); } catch (err) { this.logger.warn({ layerName: layer.layerName, err }, 'failed to sync properties, skipping to enum sync'); diff --git a/packages/synchronizer/src/sync/errors.ts b/packages/synchronizer/src/sync/errors.ts index 01b88a7..3d76d0b 100644 --- a/packages/synchronizer/src/sync/errors.ts +++ b/packages/synchronizer/src/sync/errors.ts @@ -54,6 +54,12 @@ export class StaleEnumValuesDeletionError extends SyncError { } } +export class TypeMapError extends SyncError { + public constructor(filePath: string, reason: string, cause?: unknown) { + super(`Invalid type map file ${filePath}: ${reason}`, { cause }); + } +} + export class EnumSaveError extends SyncError { public constructor(layerName: string, cause?: unknown) { super(`Failed to save enum values for ${layerName}`, { cause }); diff --git a/packages/synchronizer/src/sync/fileReader.ts b/packages/synchronizer/src/sync/fileReader.ts index c8846d6..cc1d368 100644 --- a/packages/synchronizer/src/sync/fileReader.ts +++ b/packages/synchronizer/src/sync/fileReader.ts @@ -1,10 +1,15 @@ import { createHash } from 'node:crypto'; +import type { JsonValue } from 'type-fest'; import { injectable } from 'tsyringe'; import { FsRepository } from '@common/fs/fsRepository'; import type { LayerEnums } from '@common/interfaces'; +import { parseTypeMap, type TypeMap } from './typeMap'; +import { TypeMapError } from './errors'; type AliasesFile = Record>; +export type FileAliases = Map>; + @injectable() export class FileReader { public constructor(private readonly fsRepository: FsRepository) {} @@ -20,7 +25,19 @@ export class FileReader { } } - public async readAliases(filePath: string): Promise>> { + public async readTypeMap(filePath: string): Promise { + try { + const content = await this.fsRepository.readFile(filePath, 'utf-8'); + return parseTypeMap(JSON.parse(content.toString()) as JsonValue, filePath); + } catch (err) { + if (err instanceof TypeMapError) { + throw err; + } + throw new TypeMapError(filePath, 'failed to read or parse the file', err); + } + } + + public async readAliases(filePath: string): Promise { try { const content = await this.fsRepository.readFile(filePath, 'utf-8'); const parsed = JSON.parse(content.toString()) as AliasesFile; diff --git a/packages/synchronizer/src/sync/helpers.ts b/packages/synchronizer/src/sync/helpers.ts index ef8c5bf..fad307c 100644 --- a/packages/synchronizer/src/sync/helpers.ts +++ b/packages/synchronizer/src/sync/helpers.ts @@ -1,72 +1,50 @@ import { columnType } from '@db'; import type { ColumnType } from '@db'; import type { DataSource } from 'typeorm'; +import type { JsonObject, JsonValue } from 'type-fest'; import type { InsertPropertyDTO } from '@common/interfaces'; import type { ColumnInfo } from './SyncModel'; +import type { TypeMap } from './typeMap'; -const postgresTypeMap = new Map([ - // bigint - ['bigint', columnType.bigint], - ['int8', columnType.bigint], - // real - ['real', columnType.real], - ['float4', columnType.real], - ['float8', columnType.real], - ['double precision', columnType.real], - ['numeric', columnType.real], - ['decimal', columnType.real], - // boolean - ['boolean', columnType.boolean], - ['bool', columnType.boolean], - // text - ['text', columnType.text], - ['character varying', columnType.text], - ['varchar', columnType.text], - // timestamp - ['timestamp with time zone', columnType.timestamp], - ['timestamp without time zone', columnType.timestamp], - ['timestamptz', columnType.timestamp], - ['timestamp', columnType.timestamp], - // geometry (plain, no sub-type) - ['geometry', columnType.geom], -]); +const geometryTypePattern = /^geo(?:metry|graphy)\((\w+)(?:,\s*-?\d+)?\)$/; -const geometrySubTypeMap = new Map([ - ['point', columnType.point], - ['linestring', columnType.lineString], - ['polygon', columnType.polygon], - ['multipoint', columnType.multiPoint], - ['multilinestring', columnType.multiLineString], - ['multipolygon', columnType.multiPolygon], -]); +const normalizeUdtName = (rawUdtName: string, typeMap: TypeMap): ColumnType | undefined => { + const udtName = rawUdtName.trim().toLowerCase(); -const normalizeUdtName = (udtName: string): ColumnType | undefined => { - const plain = postgresTypeMap.get(udtName); + const plain = typeMap.types.get(udtName); if (plain !== undefined) { return plain; } // Strip length/precision modifier e.g. "character varying(255)" → "character varying" - const withoutModifier = udtName.replace(/\s*\(\d+(?:,\d+)?\)$/, ''); + const withoutModifier = udtName.replace(/\s*\(\d+(?:,\s*\d+)?\)$/, ''); if (withoutModifier !== udtName) { - const stripped = postgresTypeMap.get(withoutModifier); + const stripped = typeMap.types.get(withoutModifier); if (stripped !== undefined) { return stripped; } } - const geometryMatch = /^geometry\((\w+)(?:,\d+)?\)$/i.exec(udtName); + const geometryMatch = geometryTypePattern.exec(udtName); if (geometryMatch) { - return geometrySubTypeMap.get(geometryMatch[1].toLowerCase()) ?? columnType.geom; + return typeMap.geometrySubTypes.get(geometryMatch[1]) ?? columnType.geom; } - if (udtName.startsWith('geometry')) { + if (udtName.startsWith('geometry') || udtName.startsWith('geography')) { return columnType.geom; } return undefined; }; +export const isJsonObject = (value: JsonValue | undefined): value is JsonObject => { + return typeof value === 'object' && value !== null && !Array.isArray(value); +}; + +export const excludedPropertySet = (excludeProperties?: string[]): ReadonlySet => { + return new Set((excludeProperties ?? []).map((property) => property.toLowerCase())); +}; + export const schemaOf = (dataSource: DataSource): string => { const options = dataSource.options; return 'schema' in options && typeof options.schema === 'string' ? options.schema : 'public'; @@ -75,11 +53,12 @@ export const schemaOf = (dataSource: DataSource): string => { export const columnInfosToProperties = ( columnInfos: ColumnInfo[], layerName: string, + typeMap: TypeMap, onUnknown?: (columnName: string, udtName: string) => void ): InsertPropertyDTO[] => { const result: InsertPropertyDTO[] = []; for (const { columnName, udtName } of columnInfos) { - const type = normalizeUdtName(udtName); + const type = normalizeUdtName(udtName, typeMap); if (type === undefined) { onUnknown?.(columnName, udtName); } else { diff --git a/packages/synchronizer/src/sync/typeMap.ts b/packages/synchronizer/src/sync/typeMap.ts new file mode 100644 index 0000000..ebef468 --- /dev/null +++ b/packages/synchronizer/src/sync/typeMap.ts @@ -0,0 +1,55 @@ +import type { JsonValue } from 'type-fest'; +import { columnType } from '@db'; +import type { ColumnType } from '@db'; +import { TypeMapError } from './errors'; +import { isJsonObject } from './helpers'; + +const TYPE_MAP_SECTIONS = ['types', 'geometrySubTypes'] as const; + +const supportedColumnTypes = new Set(Object.values(columnType)); + +const parseSection = (source: string, section: string, value: JsonValue | undefined): Map => { + if (!isJsonObject(value)) { + throw new TypeMapError(source, `"${section}" must be an object of postgres type name to column type`); + } + + const result = new Map(); + const invalid: string[] = []; + for (const [key, columnTypeValue] of Object.entries(value)) { + if (typeof columnTypeValue !== 'string' || !supportedColumnTypes.has(columnTypeValue)) { + invalid.push(`${section}.${key} = ${JSON.stringify(columnTypeValue)}`); + continue; + } + result.set(key.trim().toLowerCase(), columnTypeValue as ColumnType); + } + + if (invalid.length > 0) { + throw new TypeMapError( + source, + `unsupported column types [${invalid.join(', ')}], supported values are [${[...supportedColumnTypes].join(', ')}]` + ); + } + + return result; +}; + +export const parseTypeMap = (raw: JsonValue, source: string): TypeMap => { + if (!isJsonObject(raw)) { + throw new TypeMapError(source, `expected an object with the sections [${TYPE_MAP_SECTIONS.join(', ')}]`); + } + + const missing = TYPE_MAP_SECTIONS.filter((section) => !(section in raw)); + if (missing.length > 0) { + throw new TypeMapError(source, `missing sections [${missing.join(', ')}]`); + } + + return { + types: parseSection(source, 'types', raw.types), + geometrySubTypes: parseSection(source, 'geometrySubTypes', raw.geometrySubTypes), + }; +}; + +export interface TypeMap { + types: Map; + geometrySubTypes: Map; +} diff --git a/packages/synchronizer/tests/integration/syncModel/syncModel.spec.ts b/packages/synchronizer/tests/integration/syncModel/syncModel.spec.ts index 85b40d2..d93b778 100644 --- a/packages/synchronizer/tests/integration/syncModel/syncModel.spec.ts +++ b/packages/synchronizer/tests/integration/syncModel/syncModel.spec.ts @@ -21,9 +21,13 @@ import { SERVICES, SOURCE_DATA_SOURCE_PROVIDER } from '@src/common/constants'; import { S3Repository } from '@src/common/s3/s3Repository'; import { SyncModel } from '@src/sync/SyncModel'; import { schemaOf } from '@src/sync/helpers'; +import { FileReader, type FileAliases } from '@src/sync/fileReader'; +import type { TypeMap } from '@src/sync/typeMap'; const TEST_LAYER = 'test_layer'; const ENRICHMENT_ORIGIN = 'http://mock-api'; +const TYPE_MAP_FILE = './config/typeMap.json'; +const noAliases: FileAliases = new Map(); const mockEnrichmentApi = (responseBody: object, statusCode = 200): nock.Scope => nock(ENRICHMENT_ORIGIN).get(`/${TEST_LAYER}`).reply(statusCode, responseBody); @@ -66,6 +70,8 @@ const createDestinationSchema = async (dataSource: DataSource, schema: string): describe('DAL', function () { let dal: SyncModel; let enrichedDal: SyncModel; + let fileReader: FileReader; + let typeMap: TypeMap; let sourceDataSource: DataSource; let destinationDataSource: DataSource; let sourceSchema: string; @@ -88,6 +94,8 @@ describe('DAL', function () { }); dal = container.resolve(SyncModel); + fileReader = container.resolve(FileReader); + typeMap = await fileReader.readTypeMap(TYPE_MAP_FILE); sourceDataSource = container.resolve(SOURCE_DATA_SOURCE_PROVIDER); destinationDataSource = container.resolve(DESTINATION_DATA_SOURCE_PROVIDER); layerRepository = container.resolve>(LAYER_REPOSITORY_SYMBOL); @@ -110,6 +118,7 @@ describe('DAL', function () { api: 'http://mock-api/{layerName}', propertiesPath: 'fields_list', aliasField: 'display_name', + requestTimeoutMilliseconds: 10000, }; } return target.get(key); @@ -236,7 +245,7 @@ describe('DAL', function () { describe('syncProperties', function () { it('should upsert all columns from the source table into the property repository', async function () { - const affected = await dal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1); + const affected = await dal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1, typeMap, noAliases); const properties = await propertyRepository.find({ where: { layerName: TEST_LAYER } }); @@ -254,7 +263,7 @@ describe('DAL', function () { }); it('should mark enum columns with columnType.enum', async function () { - await dal.syncProperties({ layerName: TEST_LAYER, enums: ['category', 'name'] }, 1); + await dal.syncProperties({ layerName: TEST_LAYER, enums: ['category', 'name'] }, 1, typeMap, noAliases); const properties = await propertyRepository.find({ where: { layerName: TEST_LAYER } }); const categoryProp = properties.find((p) => p.property === 'category'); @@ -267,8 +276,8 @@ describe('DAL', function () { }); it('should upsert on subsequent calls without duplicating rows', async function () { - await dal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1); - await dal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1); + await dal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1, typeMap, noAliases); + await dal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1, typeMap, noAliases); const properties = await propertyRepository.find({ where: { layerName: TEST_LAYER } }); const uniqueProps = new Set(properties.map((p) => p.property)); @@ -276,12 +285,33 @@ describe('DAL', function () { expect(properties).toHaveLength(uniqueProps.size); }); + it('should not sync properties listed in excludeProperties', async function () { + await dal.syncProperties({ layerName: TEST_LAYER, enums: [], excludeProperties: ['height', 'CREATED_AT'] }, 1, typeMap, noAliases); + + const properties = await propertyRepository.find({ where: { layerName: TEST_LAYER } }); + const names = properties.map((p) => p.property); + + expect(names).not.toContain('height'); + expect(names).not.toContain('created_at'); + expect(names).toEqual(expect.arrayContaining(['id', 'name', 'active', 'shape'])); + }); + + it('should delete properties that were synced before being excluded', async function () { + await dal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1, typeMap, noAliases); + + await dal.syncProperties({ layerName: TEST_LAYER, enums: [], excludeProperties: ['height'] }, 1, typeMap, noAliases); + + const heightProp = await propertyRepository.findOne({ where: { layerName: TEST_LAYER, property: 'height' } }); + + expect(heightProp).toBeNull(); + }); + it('should normalize a parameterized geometry column to columnType.geom', async function () { await sourceDataSource.query(` ALTER TABLE "${sourceSchema}"."${TEST_LAYER}" ADD COLUMN geom_param geometry(Point,4326) `); - await dal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1); + await dal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1, typeMap, noAliases); const properties = await propertyRepository.find({ where: { layerName: TEST_LAYER } }); const geomProp = properties.find((p) => p.property === 'geom_param'); @@ -295,7 +325,7 @@ describe('DAL', function () { await sourceDataSource.query(` INSERT INTO "${sourceSchema}"."${TEST_LAYER}" (category) VALUES ('A'), ('B'), ('A'), ('C') `); - await dal.syncProperties({ layerName: TEST_LAYER, enums: ['category'] }, 1); + await dal.syncProperties({ layerName: TEST_LAYER, enums: ['category'] }, 1, typeMap, noAliases); const affected = await dal.syncEnum({ layerName: TEST_LAYER, enums: ['category'] }); @@ -307,7 +337,7 @@ describe('DAL', function () { }); it('should create an index on the enum column in the source table', async function () { - await dal.syncProperties({ layerName: TEST_LAYER, enums: ['category'] }, 1); + await dal.syncProperties({ layerName: TEST_LAYER, enums: ['category'] }, 1, typeMap, noAliases); await dal.syncEnum({ layerName: TEST_LAYER, enums: ['category'] }); const indexName = `${TEST_LAYER}_category_idx`; @@ -329,7 +359,7 @@ describe('DAL', function () { }, }); - await enrichedDal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1); + await enrichedDal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1, typeMap, noAliases); const properties = await propertyRepository.find({ where: { layerName: TEST_LAYER } }); const nameProp = properties.find((p) => p.property === 'name'); @@ -345,7 +375,7 @@ describe('DAL', function () { it('should call the enrichment API once with the resolved layer URL', async function () { const scope = mockEnrichmentApi({ fields_list: {} }); - await enrichedDal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1); + await enrichedDal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1, typeMap, noAliases); expect(scope.isDone()).toBe(true); }); @@ -353,21 +383,28 @@ describe('DAL', function () { describe('syncProperties with file aliases', function () { let tmpDir: string; + let aliasesFilePath: string; let aliasesFileDal: SyncModel; let bothDal: SyncModel; beforeAll(async function () { tmpDir = mkdtempSync(join(tmpdir(), 'dal-aliases-test-')); + aliasesFilePath = join(tmpDir, 'aliases.json'); const makeConfig = (enrichmentEnabled: boolean) => new Proxy(getConfig(), { get(target, prop) { if (prop === 'get') { return (key: string) => { - if (key === 'aliasesFile') return join(tmpDir, 'aliases.json'); if (key === 'enrichment') { return enrichmentEnabled - ? { enabled: true as const, api: 'http://mock-api/{layerName}', propertiesPath: 'fields_list', aliasField: 'display_name' } + ? { + enabled: true as const, + api: 'http://mock-api/{layerName}', + propertiesPath: 'fields_list', + aliasField: 'display_name', + requestTimeoutMilliseconds: 10000, + } : { enabled: false }; } return target.get(key); @@ -405,9 +442,9 @@ describe('DAL', function () { }); it('should apply aliases from the file when enrichment is disabled', async function () { - writeFileSync(join(tmpDir, 'aliases.json'), JSON.stringify({ [TEST_LAYER]: { name: 'File Name', height: 'File Height' } })); + writeFileSync(aliasesFilePath, JSON.stringify({ [TEST_LAYER]: { name: 'File Name', height: 'File Height' } })); - await aliasesFileDal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1); + await aliasesFileDal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1, typeMap, await fileReader.readAliases(aliasesFilePath)); const properties = await propertyRepository.find({ where: { layerName: TEST_LAYER } }); @@ -417,13 +454,13 @@ describe('DAL', function () { }); it('should override API aliases with file aliases for the same property', async function () { - writeFileSync(join(tmpDir, 'aliases.json'), JSON.stringify({ [TEST_LAYER]: { name: 'File Name' } })); + writeFileSync(aliasesFilePath, JSON.stringify({ [TEST_LAYER]: { name: 'File Name' } })); mockEnrichmentApi({ fields_list: { name: { display_name: 'API Name', type: 'TEXT' }, height: { display_name: 'API Height', type: 'REAL' } }, }); - await bothDal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1); + await bothDal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1, typeMap, await fileReader.readAliases(aliasesFilePath)); const properties = await propertyRepository.find({ where: { layerName: TEST_LAYER } }); @@ -432,11 +469,11 @@ describe('DAL', function () { }); it('should fill in aliases from the file for properties absent from the API response', async function () { - writeFileSync(join(tmpDir, 'aliases.json'), JSON.stringify({ [TEST_LAYER]: { height: 'File Height' } })); + writeFileSync(aliasesFilePath, JSON.stringify({ [TEST_LAYER]: { height: 'File Height' } })); mockEnrichmentApi({ fields_list: { name: { display_name: 'API Name', type: 'TEXT' } } }); - await bothDal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1); + await bothDal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1, typeMap, await fileReader.readAliases(aliasesFilePath)); const properties = await propertyRepository.find({ where: { layerName: TEST_LAYER } }); @@ -495,7 +532,7 @@ describe('DAL', function () { it('should leave alias unset for properties absent from the enrichment response', async function () { mockEnrichmentApi({ fields_list: { name: { display_name: 'Layer Name', type: 'TEXT' } } }); - await enrichedDal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1); + await enrichedDal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1, typeMap, noAliases); const properties = await propertyRepository.find({ where: { layerName: TEST_LAYER } }); const idProp = properties.find((p) => p.property === 'id'); @@ -507,7 +544,7 @@ describe('DAL', function () { it('should sync properties without aliases when the enrichment API returns an error response', async function () { mockEnrichmentApi({}, 500); - const affected = await enrichedDal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1); + const affected = await enrichedDal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1, typeMap, noAliases); const properties = await propertyRepository.find({ where: { layerName: TEST_LAYER } }); const idProp = properties.find((p) => p.property === 'id'); @@ -525,14 +562,14 @@ describe('DAL', function () { describe('syncProperties', function () { it('should skip columns with unsupported types and still sync the rest', async function () { await sourceDataSource.query(` - ALTER TABLE "${sourceSchema}"."${TEST_LAYER}" ADD COLUMN count integer + ALTER TABLE "${sourceSchema}"."${TEST_LAYER}" ADD COLUMN lsn pg_lsn `); - const affected = await dal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1); + const affected = await dal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1, typeMap, noAliases); const properties = await propertyRepository.find({ where: { layerName: TEST_LAYER } }); expect(affected).toBeGreaterThan(0); - expect(properties.find((p) => p.property === 'count')).toBeUndefined(); + expect(properties.find((p) => p.property === 'lsn')).toBeUndefined(); expect(properties.find((p) => p.property === 'id')).toBeDefined(); }); }); diff --git a/packages/synchronizer/tests/unit/synchronizer/helpers.spec.ts b/packages/synchronizer/tests/unit/synchronizer/helpers.spec.ts index 025f752..ce520ba 100644 --- a/packages/synchronizer/tests/unit/synchronizer/helpers.spec.ts +++ b/packages/synchronizer/tests/unit/synchronizer/helpers.spec.ts @@ -1,8 +1,14 @@ +import { readFileSync } from 'node:fs'; import { describe, it, expect, vi } from 'vitest'; +import type { JsonValue } from 'type-fest'; import { columnType } from '@map-colonies/vector-standard-db'; import { columnInfosToProperties } from '@src/sync/helpers'; +import { parseTypeMap } from '@src/sync/typeMap'; import type { ColumnInfo } from '@src/sync/SyncModel'; +const typeMapFile = './config/typeMap.json'; +const typeMap = parseTypeMap(JSON.parse(readFileSync(typeMapFile, 'utf-8')) as JsonValue, typeMapFile); + describe('columnInfosToProperties', function () { const layer = 'test_layer'; @@ -10,26 +16,44 @@ describe('columnInfosToProperties', function () { it.each([ ['character varying(255)', columnType.text], ['character varying(100)', columnType.text], + ['numeric(10,2)', columnType.real], ['bigint', columnType.bigint], ['int8', columnType.bigint], + ['integer', columnType.bigint], + ['int4', columnType.bigint], + ['smallint', columnType.bigint], + ['bigserial', columnType.bigint], ['real', columnType.real], ['float4', columnType.real], ['float8', columnType.real], ['double precision', columnType.real], + ['numeric', columnType.real], + ['decimal', columnType.real], ['boolean', columnType.boolean], ['bool', columnType.boolean], ['text', columnType.text], ['character varying', columnType.text], ['varchar', columnType.text], + ['bpchar', columnType.text], + ['uuid', columnType.text], + ['json', columnType.text], + ['jsonb', columnType.text], + ['bytea', columnType.text], + ['inet', columnType.text], + ['interval', columnType.text], + ['time without time zone', columnType.text], + ['text[]', columnType.text], ['timestamp with time zone', columnType.timestamp], ['timestamp without time zone', columnType.timestamp], ['timestamptz', columnType.timestamp], ['timestamp', columnType.timestamp], + ['date', columnType.timestamp], ['geometry', columnType.geom], + ['geography', columnType.geom], ])('should map udt "%s" to %s', function (udtName, expected) { const columnInfos: ColumnInfo[] = [{ columnName: 'col', udtName }]; - const [result] = columnInfosToProperties(columnInfos, layer); + const [result] = columnInfosToProperties(columnInfos, layer, typeMap); expect(result.type).toBe(expected); }); @@ -42,25 +66,36 @@ describe('columnInfosToProperties', function () { ['geometry(MultiLineString,4326)', columnType.multiLineString], ['geometry(MultiPolygon,4326)', columnType.multiPolygon], ['geometry(Point)', columnType.point], + ['geometry(PointZ,4326)', columnType.point], + ['geometry(MultiPolygonZM,4326)', columnType.multiPolygon], + ['geography(Point,4326)', columnType.point], + ['geography(MultiPolygon,4326)', columnType.multiPolygon], + ['geometry(GeometryCollection,4326)', columnType.geom], ])('should map "%s" to its specific GML type', function (udtName, expected) { - const [result] = columnInfosToProperties([{ columnName: 'shape', udtName }], layer); + const [result] = columnInfosToProperties([{ columnName: 'shape', udtName }], layer, typeMap); expect(result.type).toBe(expected); }); it('should map unqualified geometry to gml:GeometryPropertyType', function () { - const [result] = columnInfosToProperties([{ columnName: 'shape', udtName: 'geometry' }], layer); + const [result] = columnInfosToProperties([{ columnName: 'shape', udtName: 'geometry' }], layer, typeMap); expect(result.type).toBe(columnType.geom); }); + it('should match udt names case insensitively', function () { + const [result] = columnInfosToProperties([{ columnName: 'name', udtName: 'CHARACTER VARYING(255)' }], layer, typeMap); + + expect(result.type).toBe(columnType.text); + }); + it('should set layerName and property on every result', function () { const columnInfos: ColumnInfo[] = [ { columnName: 'name', udtName: 'text' }, { columnName: 'height', udtName: 'real' }, ]; - const result = columnInfosToProperties(columnInfos, layer); + const result = columnInfosToProperties(columnInfos, layer, typeMap); expect(result).toEqual([ { layerName: layer, property: 'name', type: columnType.text }, @@ -71,19 +106,19 @@ describe('columnInfosToProperties', function () { describe('Sad Path', function () { it('should return an empty array for no column infos', function () { - expect(columnInfosToProperties([], layer)).toEqual([]); + expect(columnInfosToProperties([], layer, typeMap)).toEqual([]); }); }); describe('Bad Path', function () { it('should map unknown geometry sub-type to gml:GeometryPropertyType', function () { - const [result] = columnInfosToProperties([{ columnName: 'shape', udtName: 'geometry(Curve,4326)' }], layer); + const [result] = columnInfosToProperties([{ columnName: 'shape', udtName: 'geometry(Curve,4326)' }], layer, typeMap); expect(result.type).toBe(columnType.geom); }); it('should return no properties for a udt name not in the type map', function () { - const result = columnInfosToProperties([{ columnName: 'data', udtName: 'json' }], layer); + const result = columnInfosToProperties([{ columnName: 'data', udtName: 'pg_lsn' }], layer, typeMap); expect(result).toHaveLength(0); }); @@ -92,15 +127,15 @@ describe('columnInfosToProperties', function () { const onUnknown = vi.fn(); const columnInfos: ColumnInfo[] = [ { columnName: 'name', udtName: 'text' }, - { columnName: 'count', udtName: 'integer' }, + { columnName: 'lsn', udtName: 'pg_lsn' }, { columnName: 'height', udtName: 'real' }, ]; - const result = columnInfosToProperties(columnInfos, layer, onUnknown); + const result = columnInfosToProperties(columnInfos, layer, typeMap, onUnknown); expect(result).toHaveLength(2); expect(result.map((p) => p.property)).toEqual(['name', 'height']); - expect(onUnknown).toHaveBeenCalledExactlyOnceWith('count', 'integer'); + expect(onUnknown).toHaveBeenCalledExactlyOnceWith('lsn', 'pg_lsn'); }); }); }); diff --git a/packages/synchronizer/tests/unit/synchronizer/typeMap.spec.ts b/packages/synchronizer/tests/unit/synchronizer/typeMap.spec.ts new file mode 100644 index 0000000..dd07458 --- /dev/null +++ b/packages/synchronizer/tests/unit/synchronizer/typeMap.spec.ts @@ -0,0 +1,58 @@ +import { readFileSync } from 'node:fs'; +import { describe, it, expect } from 'vitest'; +import type { JsonValue } from 'type-fest'; +import { columnType } from '@map-colonies/vector-standard-db'; +import { parseTypeMap } from '@src/sync/typeMap'; +import { TypeMapError } from '@src/sync/errors'; + +const typeMapFile = './config/typeMap.json'; +const source = 'test.json'; + +describe('parseTypeMap', function () { + describe('Happy Path', function () { + it('should parse the type map file shipped with the service', function () { + const typeMap = parseTypeMap(JSON.parse(readFileSync(typeMapFile, 'utf-8')) as JsonValue, typeMapFile); + + expect(typeMap.types.get('integer')).toBe(columnType.bigint); + expect(typeMap.geometrySubTypes.get('point')).toBe(columnType.point); + }); + + it('should lowercase and trim the keys', function () { + const typeMap = parseTypeMap({ types: { ' MyType ': columnType.text }, geometrySubTypes: { ['POINT']: columnType.point } }, source); + + expect(typeMap.types.get('mytype')).toBe(columnType.text); + expect(typeMap.geometrySubTypes.get('point')).toBe(columnType.point); + }); + + it('should accept empty sections', function () { + const typeMap = parseTypeMap({ types: {}, geometrySubTypes: {} }, source); + + expect(typeMap.types.size).toBe(0); + expect(typeMap.geometrySubTypes.size).toBe(0); + }); + }); + + describe('Bad Path', function () { + it.each([[null], ['not an object'], [[]]])('should throw for a non object root (%s)', function (raw) { + expect(() => parseTypeMap(raw, source)).toThrow(TypeMapError); + }); + + it('should throw when a section is missing', function () { + expect(() => parseTypeMap({ types: {} }, source)).toThrow(/missing sections \[geometrySubTypes\]/); + }); + + it('should throw when a section is not an object', function () { + expect(() => parseTypeMap({ types: [], geometrySubTypes: {} }, source)).toThrow(/"types" must be an object/); + }); + + it('should throw when a value is not a supported column type', function () { + expect(() => parseTypeMap({ types: { integer: 'xsd:banana' }, geometrySubTypes: {} }, source)).toThrow( + /unsupported column types \[types.integer = "xsd:banana"\]/ + ); + }); + + it('should throw when a value is not a string', function () { + expect(() => parseTypeMap({ types: { integer: 5 }, geometrySubTypes: {} }, source)).toThrow(TypeMapError); + }); + }); +});