feat: add type map and enrichment traces (MAPCO-11340)

helm/charts/api/templates/_helpers.tpl

@@ -63,9 +63,7 @@ Returns the tag of the chart.
 Returns the cloud provider name.
 */}}
 {{- define "api.cloudProviderFlavor" -}}
-{{- if .Values.global.cloudProvider.flavor }}
-    {{- .Values.global.cloudProvider.flavor -}}
-{{- else if .Values.cloudProvider -}}
+{{-  if .Values.cloudProvider -}}
     {{- .Values.cloudProvider.flavor | default "minikube" -}}
 {{- else -}}
     {{ "minikube" }}
@@ -76,9 +74,7 @@ Returns the cloud provider name.
 Returns the cloud provider docker registry url.
 */}}
 {{- define "api.cloudProviderDockerRegistryUrl" -}}
-{{- if .Values.global.cloudProvider.dockerRegistryUrl }}
-    {{- printf "%s/" .Values.global.cloudProvider.dockerRegistryUrl -}}
-{{- else if .Values.cloudProvider.dockerRegistryUrl -}}
+{{- if .Values.cloudProvider.dockerRegistryUrl -}}
     {{- printf "%s/" .Values.cloudProvider.dockerRegistryUrl -}}
 {{- else -}}
 {{- end -}}
@@ -88,9 +84,7 @@ Returns the cloud provider docker registry url.
 Returns the cloud provider image pull secret name.
 */}}
 {{- define "api.cloudProviderImagePullSecretName" -}}
-{{- if .Values.global.cloudProvider.imagePullSecretName }}
-    {{- .Values.global.cloudProvider.imagePullSecretName -}}
-{{- else if .Values.cloudProvider.imagePullSecretName -}}
+{{- if .Values.cloudProvider.imagePullSecretName -}}
     {{- .Values.cloudProvider.imagePullSecretName -}}
 {{- end -}}
 {{- end -}}

helm/charts/synchronizer/templates/_helpers.tpl

@@ -40,28 +40,22 @@ app.kubernetes.io/instance: {{ .Release.Name }}
 {{- end }}
 
 {{- define "synchronizer.cloudProviderFlavor" -}}
-{{- if .Values.global.cloudProvider.flavor }}
-    {{- .Values.global.cloudProvider.flavor -}}
-{{- else if .Values.cloudProvider -}}
+{{-  if .Values.cloudProvider -}}
     {{- .Values.cloudProvider.flavor | default "minikube" -}}
 {{- else -}}
     {{ "minikube" }}
 {{- end -}}
 {{- end -}}
 
 {{- define "synchronizer.cloudProviderDockerRegistryUrl" -}}
-{{- if .Values.global.cloudProvider.dockerRegistryUrl }}
-    {{- printf "%s/" .Values.global.cloudProvider.dockerRegistryUrl -}}
-{{- else if .Values.cloudProvider.dockerRegistryUrl -}}
+{{-  if .Values.cloudProvider.dockerRegistryUrl -}}
     {{- printf "%s/" .Values.cloudProvider.dockerRegistryUrl -}}
 {{- else -}}
 {{- end -}}
 {{- end -}}
 
 {{- define "synchronizer.cloudProviderImagePullSecretName" -}}
-{{- if .Values.global.cloudProvider.imagePullSecretName }}
-    {{- .Values.global.cloudProvider.imagePullSecretName -}}
-{{- else if .Values.cloudProvider.imagePullSecretName -}}
+{{-  if .Values.cloudProvider.imagePullSecretName -}}
     {{- .Values.cloudProvider.imagePullSecretName -}}
 {{- end -}}
 {{- end -}}

helm/charts/synchronizer/templates/configmap.yaml

@@ -36,6 +36,10 @@ data:
 
   ALIASES_FILE_PATH: {{ .Values.env.aliasesFile | quote }}
 
+  {{- if .Values.typeMap.enabled }}
+  TYPE_MAP_FILE_PATH: {{ .Values.env.typeMapFile | quote }}
+  {{- end }}
+
   {{- with .Values.dbs.s3 }}
   S3_ENDPOINT: {{ .endpoint | quote }}
   S3_REGION: {{ .region | quote }}
@@ -54,6 +58,7 @@ data:
   ENRICHMENT_API_URL: {{ .api | quote }}
   ENRICHMENT_PROPERTIES_PATH: {{ .propertiesPath | quote }}
   ENRICHMENT_ALIAS_FIELD: {{ .aliasField | quote }}
+  ENRICHMENT_REQUEST_TIMEOUT_MILLISECONDS: {{ .requestTimeoutMilliseconds | quote }}
   {{- end }}
   {{- end }}
   {{- include "synchronizer.dbEnvBlock" (dict "prefix" "SOURCE_DB" "db" .Values.dbs.sourceDb) | nindent 2 }}

helm/charts/synchronizer/templates/deployment.yaml

@@ -96,6 +96,10 @@ spec:
               mountPath: {{ .Values.env.layersFile | dir | quote }}
             - name: aliases-config
               mountPath: {{ .Values.env.aliasesFile | dir | quote }}
+            {{- if .Values.typeMap.enabled }}
+            - name: typemap-config
+              mountPath: {{ .Values.env.typeMapFile | dir | quote }}
+            {{- end }}
             - name: s3-downloads
               mountPath: /usr/src/app/packages/synchronizer/dist/src/common/s3/downloads
             {{- if .Values.dbs.sourceDb.sslAuth.enabled }}
@@ -120,6 +124,11 @@ spec:
         - name: aliases-config
           configMap:
             name: {{ include "synchronizer.fullname" . }}-aliases
+        {{- if .Values.typeMap.enabled }}
+        - name: typemap-config
+          configMap:
+            name: {{ include "synchronizer.fullname" . }}-typemap
+        {{- end }}
         - name: s3-downloads
           emptyDir: {}
         {{- if .Values.dbs.sourceDb.sslAuth.enabled }}

helm/charts/synchronizer/templates/typemap-configmap.yaml

@@ -0,0 +1,10 @@
+{{- if and .Values.enabled .Values.typeMap.enabled -}}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "synchronizer.fullname" . }}-typemap
+  labels:
+    {{- include "synchronizer.labels" . | nindent 4 }}
+data:
+  {{ .Values.env.typeMapFile | base }}: {{ .Values.typeMap.data | toPrettyJson | quote }}
+{{- end }}

helm/charts/synchronizer/values.yaml

@@ -49,6 +49,12 @@ layers:
 
 aliases: {}
 
+typeMap:
+  enabled: false
+  data:
+    types: {}
+    geometrySubTypes: {}
+
 env:
   port: 80
   targetPort: 8080
@@ -66,11 +72,13 @@ env:
   indexNameFormat: '{layerName}_{column}_idx'
   layersFile: /usr/src/app/packages/synchronizer/dist/config/layers/layers.json
   aliasesFile: /usr/src/app/packages/synchronizer/dist/config/aliases/aliases.json
+  typeMapFile: /usr/src/app/packages/synchronizer/dist/config/typeMap/typeMap.json
   enrichment:
     enabled: false
     api: ""
     propertiesPath: ""
     aliasField: ""
+    requestTimeoutMilliseconds: 5000
 
 dbs:
   s3:

helm/values.yaml

@@ -17,7 +17,7 @@ mclabels: &mclabels
     path: /metrics
 
 cloudProvider:
-  flavor: openshift
+  flavor: &cloudProviderFlavor openshift
   dockerRegistryUrl: &dockerRegistryUrl ''
   imagePullSecretName: &pullSecret ''
 
@@ -44,13 +44,14 @@ api:
   initialDelaySeconds: 60
 
   cloudProvider:
+    flavor: *cloudProviderFlavor
     dockerRegistryUrl: *dockerRegistryUrl
     imagePullSecretName: *pullSecret
 
   image:
     repository: standard-api
     pullPolicy: Always
-    tag: "v1.0.0"
+    tag: "v1.1.0"
 
   startupProbe:
     enabled: true
@@ -153,21 +154,28 @@ synchronizer:
     path: /liveness
 
   cloudProvider:
+    flavor: *cloudProviderFlavor
     dockerRegistryUrl: *dockerRegistryUrl
     imagePullSecretName: *pullSecret
 
   image:
     repository: standard-synchronizer
     pullPolicy: Always
-    tag: "v1.0.0"
+    tag: "v1.2.0"
 
   layers:
   - layerName: buildings_polygon
     enums: [ code, attribute ]
 
   aliases:
     "*":
-      number_of_floors: "number_of_floors"
+      num_floors: "number_of_floors"
+
+  typeMap:
+    enabled: false
+    data:
+      types: {}
+      geometrySubTypes: {}
 
   env:
     port: 80
@@ -186,11 +194,13 @@ synchronizer:
     indexNameFormat: '{layerName}_{column}_idx'
     layersFile: /usr/src/app/packages/synchronizer/dist/config/layers/layers.json
     aliasesFile: /usr/src/app/packages/synchronizer/dist/config/aliases/aliases.json
+    typeMapFile: /usr/src/app/packages/synchronizer/dist/config/typeMap.json
     enrichment:
       enabled: false
       api: ""
       propertiesPath: ""
       aliasField: ""
+      requestTimeoutMilliseconds: 5000
 
   dbs:
     s3:

package.json

@@ -23,7 +23,7 @@
     "typescript": "^5.9.3"
   },
   "dependencies": {
-    "@map-colonies/schemas": "1.25.1",
+    "@map-colonies/schemas": "https://ghatmpstorage.blob.core.windows.net/npm-packages/schemas-4b2c908bb50bda87cd8081773704a9718de4c2e6.tgz",
     "lerna": "^6.6.2"
   }
 }

packages/synchronizer/config/default.json

@@ -5,6 +5,7 @@
   },
   "layersFile": "./config/layers.json",
   "aliasesFile": "./config/aliases.json",
+  "typeMapFile": "./config/typeMap.json",
   "schedule": "*/1 * * * *",
   "indexNameFormat": "{layerName}_{column}_idx",
   "enrichment": {

packages/synchronizer/config/layers.json

@@ -1,6 +1,7 @@
 [
   {
     "layerName": "buildings_polygon",
+    "excludeProperties": ["id"],
     "enums": ["code", "attribute"]
   }
 ]

packages/synchronizer/config/test.json

@@ -11,6 +11,7 @@
     "enabled": true,
     "api": "https://example.com/{layerName}",
     "propertiesPath": "fields_list",
-    "aliasField": "display_name"
+    "aliasField": "display_name",
+    "requestTimeoutMilliseconds": 10000
   }
 }

packages/synchronizer/config/typeMap.json

@@ -0,0 +1,114 @@
+{
+  "types": {
+    "bigint": "xsd:long",
+    "int8": "xsd:long",
+    "bigserial": "xsd:long",
+    "serial8": "xsd:long",
+    "integer": "xsd:long",
+    "int": "xsd:long",
+    "int4": "xsd:long",
+    "serial": "xsd:long",
+    "serial4": "xsd:long",
+    "smallint": "xsd:long",
+    "int2": "xsd:long",
+    "smallserial": "xsd:long",
+    "serial2": "xsd:long",
+    "oid": "xsd:long",
+
+    "real": "xsd:double",
+    "float4": "xsd:double",
+    "float8": "xsd:double",
+    "float": "xsd:double",
+    "double precision": "xsd:double",
+    "numeric": "xsd:double",
+    "decimal": "xsd:double",
+
+    "boolean": "xsd:boolean",
+    "bool": "xsd:boolean",
+
+    "text": "xsd:string",
+    "character varying": "xsd:string",
+    "varchar": "xsd:string",
+    "character": "xsd:string",
+    "char": "xsd:string",
+    "bpchar": "xsd:string",
+    "name": "xsd:string",
+    "citext": "xsd:string",
+    "uuid": "xsd:string",
+    "json": "xsd:string",
+    "jsonb": "xsd:string",
+    "xml": "xsd:string",
+    "bytea": "xsd:string",
+    "money": "xsd:string",
+    "inet": "xsd:string",
+    "cidr": "xsd:string",
+    "macaddr": "xsd:string",
+    "macaddr8": "xsd:string",
+    "interval": "xsd:string",
+    "time": "xsd:string",
+    "time without time zone": "xsd:string",
+    "time with time zone": "xsd:string",
+    "timetz": "xsd:string",
+    "tsvector": "xsd:string",
+    "tsquery": "xsd:string",
+    "text[]": "xsd:string",
+    "character varying[]": "xsd:string",
+    "varchar[]": "xsd:string",
+    "integer[]": "xsd:string",
+    "bigint[]": "xsd:string",
+    "numeric[]": "xsd:string",
+    "boolean[]": "xsd:string",
+    "uuid[]": "xsd:string",
+    "jsonb[]": "xsd:string",
+
+    "timestamp": "xsd:dateTime",
+    "timestamp with time zone": "xsd:dateTime",
+    "timestamp without time zone": "xsd:dateTime",
+    "timestamptz": "xsd:dateTime",
+    "date": "xsd:dateTime",
+
+    "geometry": "gml:GeometryPropertyType",
+    "geography": "gml:GeometryPropertyType"
+  },
+  "geometrySubTypes": {
+    "point": "gml:PointPropertyType",
+    "pointz": "gml:PointPropertyType",
+    "pointm": "gml:PointPropertyType",
+    "pointzm": "gml:PointPropertyType",
+
+    "linestring": "gml:LineStringPropertyType",
+    "linestringz": "gml:LineStringPropertyType",
+    "linestringm": "gml:LineStringPropertyType",
+    "linestringzm": "gml:LineStringPropertyType",
+
+    "polygon": "gml:PolygonPropertyType",
+    "polygonz": "gml:PolygonPropertyType",
+    "polygonm": "gml:PolygonPropertyType",
+    "polygonzm": "gml:PolygonPropertyType",
+
+    "multipoint": "gml:MultiPointPropertyType",
+    "multipointz": "gml:MultiPointPropertyType",
+    "multipointm": "gml:MultiPointPropertyType",
+    "multipointzm": "gml:MultiPointPropertyType",
+
+    "multilinestring": "gml:MultiLineStringPropertyType",
+    "multilinestringz": "gml:MultiLineStringPropertyType",
+    "multilinestringm": "gml:MultiLineStringPropertyType",
+    "multilinestringzm": "gml:MultiLineStringPropertyType",
+
+    "multipolygon": "gml:MultiPolygonPropertyType",
+    "multipolygonz": "gml:MultiPolygonPropertyType",
+    "multipolygonm": "gml:MultiPolygonPropertyType",
+    "multipolygonzm": "gml:MultiPolygonPropertyType",
+
+    "geometrycollection": "gml:GeometryPropertyType",
+    "circularstring": "gml:GeometryPropertyType",
+    "compoundcurve": "gml:GeometryPropertyType",
+    "curvepolygon": "gml:GeometryPropertyType",
+    "multicurve": "gml:GeometryPropertyType",
+    "multisurface": "gml:GeometryPropertyType",
+    "polyhedralsurface": "gml:GeometryPropertyType",
+    "tin": "gml:GeometryPropertyType",
+    "triangle": "gml:GeometryPropertyType"
+  }
+}