Replace Tableland on-chain device definitions with the R2 catalog

charts/device-definitions-api/templates/secret.yaml

@@ -6,17 +6,14 @@ metadata:
 spec:
   data:
   - remoteRef:
-      key: {{ .Release.Namespace }}/definitions/ethereum/rpc_url
-    secretKey: ETHEREUM_RPC_URL
+      key: {{ .Release.Namespace }}/definitions/worker/token
+    secretKey: DEFINITIONS_WORKER_TOKEN
   - remoteRef:
       key: {{ .Release.Namespace }}/definitions/aws/access_key_id
     secretKey: AWS_ACCESS_KEY_ID
   - remoteRef:
       key: {{ .Release.Namespace }}/definitions/aws/secret_access_key
     secretKey: AWS_SECRET_ACCESS_KEY
-  - remoteRef:
-      key: {{ .Release.Namespace }}/definitions/kms/key_id
-    secretKey: KMS_KEY_ID
   - remoteRef:
       key: {{ .Release.Namespace }}/definitions/db/host
     secretKey: DB_HOST
@@ -77,9 +74,6 @@ spec:
   - remoteRef:
       key: {{ .Release.Namespace }}/definitions/typesense/apiurl
     secretKey: SEARCH_SERVICE_API_URL
-  - remoteRef:
-      key: {{ .Release.Namespace }}/definitions/polygonscan/apikey
-    secretKey: POLYGON_SCAN_API_KEY
   - remoteRef:
       key: {{ .Release.Namespace }}/definitions/googlesheets/credentials
     secretKey: GOOGLE_SHEETS_CREDENTIALS

charts/device-definitions-api/values-prod.yaml

@@ -20,9 +20,8 @@ env:
   REDIS_CLUSTER_MODE: true
   REDIS_TLS: true
   KAFKA_BROKERS: kafka-prod-dimo-kafka-kafka-brokers:9092
-  ETHEREUM_SEND_TRANSACTION: true
-  TABLELAND_API_GATEWAY: http://tableland-node-prod.prod.svc.cluster.local:8080/
-  ETHEREUM_REGISTRY_ADDRESS: '0xFA8beC73cebB9D88FF88a2f75E7D7312f2Fd39EC'
+  DEFINITIONS_CATALOG_URL: https://definitions.dimo.org
+  DEFINITIONS_WORKER_URL: https://definitions.dimo.org
   SEARCH_SERVICE_DEVICE_DEFINITION_INDEX: definitions_prod
   IDENTITY_API_URL: https://identity-api.dimo.zone/query
 ingress:

charts/device-definitions-api/values.yaml

@@ -51,9 +51,8 @@ env:
   VINCARIO_API_URL: https://api.vindecoder.eu/3.2
   DAT_GROUP_URL: https://www.datgroup.com/myClaim/soap/v2/VehicleIdentificationService
   PRIVATE_KEY_MODE: false
-  ETHEREUM_SEND_TRANSACTION: true
-  TABLELAND_API_GATEWAY: http://tableland-node-dev.dev.svc.cluster.local:8080/
-  ETHEREUM_REGISTRY_ADDRESS: '0x5eAA326fB2fc97fAcCe6A79A304876daD0F2e96c'
+  DEFINITIONS_CATALOG_URL: https://definitions.dev.dimo.org
+  DEFINITIONS_WORKER_URL: https://definitions.dev.dimo.org
   SEARCH_SERVICE_DEVICE_DEFINITION_INDEX: definitions_dev
   IDENTITY_API_URL: https://identity-api.dev.dimo.zone/query
 service:

cmd/device-definitions-api/add_vin.go

@@ -14,8 +14,6 @@ import (
 
 	"github.com/DIMO-Network/device-definitions-api/internal/core/common"
 	"github.com/DIMO-Network/device-definitions-api/internal/infrastructure/gateways"
-	"github.com/DIMO-Network/device-definitions-api/internal/infrastructure/sender"
-	"github.com/ethereum/go-ethereum/ethclient"
 
 	"github.com/DIMO-Network/device-definitions-api/internal/config"
 	"github.com/DIMO-Network/device-definitions-api/internal/infrastructure/db/models"
@@ -32,8 +30,6 @@ import (
 type addVINCmd struct {
 	logger   zerolog.Logger
 	settings config.Settings
-
-	sender sender.Sender
 }
 
 func (*addVINCmd) Name() string     { return "addvin" }
@@ -61,16 +57,7 @@ func (p *addVINCmd) Execute(ctx context.Context, _ *flag.FlagSet, _ ...interface
 	pdb := db.NewDbConnectionFromSettings(ctx, &p.settings.DB, true)
 	pdb.WaitForDB(p.logger)
 
-	ethClient, err := ethclient.Dial(p.settings.EthereumRPCURL.String())
-	if err != nil {
-		p.logger.Fatal().Err(err).Msg("Failed to create Ethereum client.")
-	}
-
-	chainID, err := ethClient.ChainID(ctx)
-	if err != nil {
-		p.logger.Fatal().Err(err).Msg("Couldn't retrieve chain id.")
-	}
-	onChainSvc := gateways.NewDeviceDefinitionOnChainService(&p.settings, &p.logger, ethClient, chainID, p.sender, pdb.DBS)
+	catalogSvc := gateways.NewDeviceDefinitionCatalogService(&p.settings, &p.logger)
 
 	vinDecodeNumber, err := models.FindVinNumber(ctx, pdb.DBS().Reader, vin)
 	if err != nil && !errors.Is(err, sql.ErrNoRows) {
@@ -117,18 +104,24 @@ func (p *addVINCmd) Execute(ctx context.Context, _ *flag.FlagSet, _ ...interface
 		return subcommands.ExitFailure
 	}
 
-	manufacturer, err := onChainSvc.GetManufacturer(stringutils.SlugString(wmi.ManufacturerName))
+	manufacturer, err := catalogSvc.GetManufacturer(stringutils.SlugString(wmi.ManufacturerName))
 	if err != nil {
 		fmt.Println(err.Error())
 		return subcommands.ExitFailure
 	}
 	definitionID := common.DeviceDefinitionSlug(stringutils.SlugString(wmi.ManufacturerName), stringutils.SlugString(model), int16(vinNumber.Year))
-	deviceDefinition, err := onChainSvc.GetDefinitionTableland(ctx, big.NewInt(int64(manufacturer.TokenID)), definitionID)
+	deviceDefinition, err := catalogSvc.GetDefinition(ctx, big.NewInt(int64(manufacturer.TokenID)), definitionID)
 
 	if err != nil {
 		fmt.Println(err.Error() + " " + model + " " + strconv.Itoa(vinNumber.Year))
 		return subcommands.ExitFailure
 	}
+	if deviceDefinition == nil {
+		// (nil, nil) is returned both for a missing definition and for one
+		// owned by a different manufacturer; dereferencing panics.
+		fmt.Println("no device definition found for " + definitionID)
+		return subcommands.ExitFailure
+	}
 	vinNumber.DefinitionID = deviceDefinition.ID
 
 	err = vinNumber.Insert(ctx, pdb.DBS().Writer, boil.Infer())

cmd/device-definitions-api/add_vins_csv.go

@@ -13,7 +13,6 @@ import (
 	"github.com/DIMO-Network/device-definitions-api/internal/config"
 	"github.com/DIMO-Network/device-definitions-api/internal/infrastructure/db/models"
 	"github.com/DIMO-Network/device-definitions-api/internal/infrastructure/gateways"
-	"github.com/DIMO-Network/device-definitions-api/internal/infrastructure/sender"
 	"github.com/DIMO-Network/shared/pkg/db"
 	"github.com/DIMO-Network/shared/pkg/logfields"
 	vinutils "github.com/DIMO-Network/shared/pkg/vin"
@@ -28,7 +27,6 @@ type addVINsCSVCmd struct {
 	logger   zerolog.Logger
 	settings config.Settings
 
-	sender   sender.Sender
 	identity gateways.IdentityAPI
 }
 

cmd/device-definitions-api/bulk_update_powertrain.go

@@ -3,21 +3,17 @@ package main
 import (
 	"context"
 	"encoding/csv"
-	"encoding/json"
 	"flag"
 	"fmt"
 	"os"
 	"strings"
 	"time"
 
-	"github.com/DIMO-Network/device-definitions-api/internal/contracts"
 	"github.com/DIMO-Network/device-definitions-api/internal/core/common"
-	"github.com/DIMO-Network/device-definitions-api/internal/core/models"
 	"github.com/DIMO-Network/device-definitions-api/internal/infrastructure/gateways"
-	"github.com/DIMO-Network/device-definitions-api/internal/infrastructure/sender"
-	"github.com/ethereum/go-ethereum/ethclient"
 
 	"github.com/DIMO-Network/device-definitions-api/internal/config"
+	"github.com/DIMO-Network/device-definitions-api/internal/core/models"
 	"github.com/DIMO-Network/shared/pkg/db"
 	"github.com/google/subcommands"
 	"github.com/rs/zerolog"
@@ -26,8 +22,6 @@ import (
 type bulkUpdatePowertrain struct {
 	logger   zerolog.Logger
 	settings config.Settings
-
-	sender sender.Sender
 }
 
 func (*bulkUpdatePowertrain) Name() string { return "bulk-update-powertrain" }
@@ -66,16 +60,7 @@ func (p *bulkUpdatePowertrain) Execute(ctx context.Context, _ *flag.FlagSet, _ .
 	pdb := db.NewDbConnectionFromSettings(ctx, &p.settings.DB, true)
 	pdb.WaitForDB(p.logger)
 
-	ethClient, err := ethclient.Dial(p.settings.EthereumRPCURL.String())
-	if err != nil {
-		p.logger.Fatal().Err(err).Msg("Failed to create Ethereum client.")
-	}
-
-	chainID, err := ethClient.ChainID(ctx)
-	if err != nil {
-		p.logger.Fatal().Err(err).Msg("Couldn't retrieve chain id.")
-	}
-	onChainSvc := gateways.NewDeviceDefinitionOnChainService(&p.settings, &p.logger, ethClient, chainID, p.sender, pdb.DBS)
+	catalogSvc := gateways.NewDeviceDefinitionCatalogService(&p.settings, &p.logger)
 
 	notFoundDefinitions := make([]string, 0)
 
@@ -101,7 +86,9 @@ func (p *bulkUpdatePowertrain) Execute(ctx context.Context, _ *flag.FlagSet, _ .
 		}
 		fmt.Printf("DefinitionID: %s, Powertrain: %s\n", definitionID, powertrain)
 
-		deviceDefinition, manufID, err := onChainSvc.GetDefinitionByID(ctx, definitionID)
+		// Fresh: this reads, mutates metadata and writes it back, so a CDN-cached
+		// base would silently drop any edit made in the last day.
+		deviceDefinition, manufID, err := catalogSvc.GetDefinitionByIDFresh(ctx, definitionID)
 		if err != nil {
 			fmt.Printf("%s: Error getting device definition: %v\n", definitionID, err)
 			notFoundDefinitions = append(notFoundDefinitions, definitionID)
@@ -113,13 +100,13 @@ func (p *bulkUpdatePowertrain) Execute(ctx context.Context, _ *flag.FlagSet, _ .
 			continue
 		}
 
-		manufName, err := onChainSvc.GetManufacturerNameByID(ctx, manufID)
+		manufName, err := catalogSvc.GetManufacturerNameByID(ctx, manufID)
 		if err != nil {
 			fmt.Printf("%s: Error getting manufacturer name: %v\n", manufID, err)
 			continue
 		}
 		set := false
-		if deviceDefinition.Metadata != nil {
+		if deviceDefinition.Metadata == nil {
 			deviceDefinition.Metadata = &models.DeviceDefinitionMetadata{
 				DeviceAttributes: make([]models.DeviceTypeAttribute, 0),
 			}
@@ -138,16 +125,15 @@ func (p *bulkUpdatePowertrain) Execute(ctx context.Context, _ *flag.FlagSet, _ .
 				Value: powertrain,
 			})
 		}
-		md, _ := json.Marshal(deviceDefinition.Metadata)
-		updateContract := contracts.DeviceDefinitionUpdateInput{
-			Id:         deviceDefinition.ID,
-			Metadata:   string(md),
-			Ksuid:      deviceDefinition.KSUID,
+		updateInput := models.DeviceDefinitionUpdateInput{
+			ID:         deviceDefinition.ID,
+			Metadata:   deviceDefinition.Metadata,
+			KSUID:      deviceDefinition.KSUID,
 			DeviceType: deviceDefinition.DeviceType,
 			ImageURI:   deviceDefinition.ImageURI,
 		}
 
-		update, err := onChainSvc.Update(ctx, manufName, updateContract)
+		update, err := catalogSvc.Update(ctx, manufName, updateInput)
 		if err != nil {
 			fmt.Printf("%s: Error updating device definition: %v\n", definitionID, err)
 			return subcommands.ExitFailure

cmd/device-definitions-api/decode_vin.go

@@ -16,7 +16,6 @@ import (
 	vinutil "github.com/DIMO-Network/shared/pkg/vin"
 	"github.com/aarondl/null/v8"
 	"github.com/aarondl/sqlboiler/v4/boil"
-	"github.com/ethereum/go-ethereum/ethclient"
 
 	"github.com/goccy/go-json"
 
@@ -254,7 +253,7 @@ func readVINFile(filename string) ([]string, error) {
 	return values, nil
 }
 
-func instantiateVINDecodingSvc(ctx context.Context, settings *config.Settings, logger *zerolog.Logger, pdb db.Store) services.VINDecodingService {
+func instantiateVINDecodingSvc(_ context.Context, settings *config.Settings, logger *zerolog.Logger, pdb db.Store) services.VINDecodingService {
 	datAPI := gateways.NewDATGroupAPIService(settings, logger)
 	drivlyAPI := gateways.NewDrivlyAPIService(settings)
 	vincarioAPI := gateways.NewVincarioAPIService(settings, logger)
@@ -265,21 +264,7 @@ func instantiateVINDecodingSvc(ctx context.Context, settings *config.Settings, l
 		return services.NewVINDecodingService(drivlyAPI, vincarioAPI, nil, logger, nil, datAPI, pdb.DBS, jp17vinAPI, carvxAPI, elevaAPI)
 	}
 
-	send, err := createSender(ctx, settings, logger)
-	if err != nil {
-		logger.Fatal().Err(err).Msg("Failed to create sender.")
-	}
-
-	ethClient, err := ethclient.Dial(settings.EthereumRPCURL.String())
-	if err != nil {
-		logger.Fatal().Err(err).Msg("Failed to create Ethereum client.")
-	}
-
-	chainID, err := ethClient.ChainID(ctx)
-	if err != nil {
-		logger.Fatal().Err(err).Msg("Couldn't retrieve chain id.")
-	}
-	deviceDefinitionOnChainService := gateways.NewDeviceDefinitionOnChainService(settings, logger, ethClient, chainID, send, pdb.DBS)
+	deviceDefinitionCatalogService := gateways.NewDeviceDefinitionCatalogService(settings, logger)
 
-	return services.NewVINDecodingService(drivlyAPI, vincarioAPI, nil, logger, deviceDefinitionOnChainService, datAPI, pdb.DBS, jp17vinAPI, carvxAPI, elevaAPI)
+	return services.NewVINDecodingService(drivlyAPI, vincarioAPI, nil, logger, deviceDefinitionCatalogService, datAPI, pdb.DBS, jp17vinAPI, carvxAPI, elevaAPI)
 }

cmd/device-definitions-api/delete_dd.go

@@ -7,13 +7,10 @@ import (
 	"fmt"
 	"os"
 	"strings"
-	"time"
 
 	"github.com/DIMO-Network/device-definitions-api/internal/config"
-	dd_common "github.com/DIMO-Network/device-definitions-api/internal/core/common"
 	"github.com/DIMO-Network/device-definitions-api/internal/infrastructure/gateways"
 	"github.com/DIMO-Network/shared/pkg/db"
-	"github.com/ethereum/go-ethereum/ethclient"
 	"github.com/google/subcommands"
 	"github.com/rs/zerolog"
 )
@@ -52,50 +49,17 @@ func (p *deleteDefinition) Execute(ctx context.Context, _ *flag.FlagSet, _ ...in
 
 	pdb := db.NewDbConnectionFromSettings(ctx, &p.settings.DB, true)
 	pdb.WaitForDB(p.logger)
-	send, err := createSender(ctx, &p.settings, &p.logger)
-	if err != nil {
-		p.logger.Fatal().Err(err).Msg("Failed to create sender.")
-	}
 
-	ethClient, err := ethclient.Dial(p.settings.EthereumRPCURL.String())
-	if err != nil {
-		p.logger.Fatal().Err(err).Msg("Failed to create Ethereum client.")
-	}
-
-	chainID, err := ethClient.ChainID(ctx)
-	if err != nil {
-		p.logger.Fatal().Err(err).Msg("Couldn't retrieve chain id.")
-	}
-	deviceDefinitionOnChainService := gateways.NewDeviceDefinitionOnChainService(&p.settings, &p.logger, ethClient, chainID, send, pdb.DBS)
+	deviceDefinitionCatalogService := gateways.NewDeviceDefinitionCatalogService(&p.settings, &p.logger)
 
 	id := os.Args[len(os.Args)-1]
 
-	trx, err := deviceDefinitionOnChainService.Delete(ctx, manufacturer, id)
+	deleted, err := deviceDefinitionCatalogService.Delete(ctx, manufacturer, id)
 	if err != nil {
 		p.logger.Fatal().Err(err).Msg("Failed to delete.")
 	}
-
-	if len(*trx) > 0 {
-		trxFinished := false
-		loops := 0
-		for !trxFinished {
-			loops++
-			time.Sleep(time.Second * 2)
-			trxFinished, err = dd_common.CheckTransactionStatus(*trx, p.settings.PolygonScanAPIKey, !p.settings.IsProd())
-			if err != nil {
-				fmt.Println("Error checking transaction status: ", err)
-			}
-			fmt.Println("Transaction status: ", trxFinished)
-			if loops > 10 {
-				// get device definition from on chain to see if maybe got created but trx still showing false
-				onchainDD, _, err := deviceDefinitionOnChainService.GetDefinitionByID(ctx, id)
-				fmt.Println("onchainDD: ", onchainDD, err)
-				if onchainDD != nil {
-					break
-				}
-			}
-		}
-	}
+	// The worker delete is synchronous; nothing to poll.
+	fmt.Println("Deleted device definition: ", *deleted)
 
 	return subcommands.ExitSuccess
 }