UserStore: Add migration logic and migration interpreter

cassandra-schema.cql

@@ -738,7 +738,7 @@ CREATE TABLE brig_test.rich_info (
     AND crc_check_chance = 1.0
     AND default_time_to_live = 0
     AND extensions = {}
-    AND gc_grace_seconds = 864000
+    AND gc_grace_seconds = 86400
     AND max_index_interval = 2048
     AND memtable_flush_period_in_ms = 0
     AND min_index_interval = 128
@@ -848,7 +848,7 @@ CREATE TABLE brig_test.service_team (
     AND crc_check_chance = 1.0
     AND default_time_to_live = 0
     AND extensions = {}
-    AND gc_grace_seconds = 864000
+    AND gc_grace_seconds = 86400
     AND max_index_interval = 2048
     AND memtable_flush_period_in_ms = 0
     AND min_index_interval = 128
@@ -874,7 +874,7 @@ CREATE TABLE brig_test.service_user (
     AND crc_check_chance = 1.0
     AND default_time_to_live = 0
     AND extensions = {}
-    AND gc_grace_seconds = 864000
+    AND gc_grace_seconds = 86400
     AND max_index_interval = 2048
     AND memtable_flush_period_in_ms = 0
     AND min_index_interval = 128
@@ -1088,7 +1088,7 @@ CREATE TABLE brig_test.user (
     AND crc_check_chance = 1.0
     AND default_time_to_live = 0
     AND extensions = {}
-    AND gc_grace_seconds = 864000
+    AND gc_grace_seconds = 86400
     AND max_index_interval = 2048
     AND memtable_flush_period_in_ms = 0
     AND min_index_interval = 128
@@ -1137,7 +1137,7 @@ CREATE TABLE brig_test.user_handle (
     AND crc_check_chance = 1.0
     AND default_time_to_live = 0
     AND extensions = {}
-    AND gc_grace_seconds = 864000
+    AND gc_grace_seconds = 86400
     AND max_index_interval = 2048
     AND memtable_flush_period_in_ms = 0
     AND min_index_interval = 128

changelog.d/2-features/user-pg-migration

@@ -0,0 +1 @@
+Support migrating user data to postgresql from cassandra

charts/wire-server/templates/background-worker/configmap.yaml

@@ -84,6 +84,7 @@ data:
     migrateConversationCodes: {{ .migrateConversationCodes }}
     migrateTeamFeatures: {{ .migrateTeamFeatures }}
     migrateDomainRegistration: {{ .migrateDomainRegistration }}
+    migrateUsers: {{ .migrateUsers }}
     migrationOptions:
 {{ toYaml .migrationOptions | indent 6 }}
 

charts/wire-server/values.yaml

@@ -1026,6 +1026,10 @@ background-worker:
     # It's important to set `settings.postgresMigration.domainRegistration` to `migration-to-postgresql`
     # before starting the migration.
     migrateDomainRegistration: false
+    # This will start the migration of users
+    # It's important to set `settings.postgresMigration.users` to `migration-to-postgresql`
+    # before starting the migration.
+    migrateUsers: false
 
     backendNotificationPusher:
       pushBackoffMinWait: 10000 # in microseconds, so 10ms

deploy/dockerephemeral/docker-compose.yaml

@@ -290,7 +290,7 @@ services:
       POSTGRES_PASSWORD: "posty-the-gres"
       POSTGRES_USER: "wire-server"
       POSTGRES_DB: "backendA"
-    command: postgres -c max_connections=150
+    command: postgres -c max_connections=1000
 
   cassandra:
     container_name: demo_wire_cassandra

docs/src/developer/reference/config-options.md

@@ -2187,6 +2187,7 @@ The current settings and their background-worker flags are:
 - `conversationCodes` -> `migrateConversationCodes`
 - `teamFeatures` -> `migrateTeamFeatures`
 - `domainRegistration` -> `migrateDomainRegistration`
+- `user` -> `migrateUsers`
 
 **Migration pattern per migration setting**
 
@@ -2205,13 +2206,15 @@ The current settings and their background-worker flags are:
          conversation: migration-to-postgresql
          conversationCodes: migration-to-postgresql
          teamFeatures: migration-to-postgresql
-         domainRegistration: cassandra
+         domainRegistration: migration-to-postgresql
+         user: migration-to-postgresql
    background-worker:
      config:
        migrateConversations: false
        migrateConversationCodes: false
        migrateTeamFeatures: false
        migrateDomainRegistration: false
+       migrateUsers: false
    ```
 
    This change should restart the affected pods, and new writes will follow the
@@ -2226,6 +2229,7 @@ The current settings and their background-worker flags are:
        migrateConversationCodes: true
        migrateTeamFeatures: true
        migrateDomainRegistration: true
+       migrateUsers: true
    ```
 
    During migration, Cassandra rows are not deleted. Writes and migration share
@@ -2241,6 +2245,16 @@ The current settings and their background-worker flags are:
    - `conversationCodes`: `wire_conv_codes_migration_finished`
    - `teamFeatures`: `wire_team_features_migration_finished`
    - `domainRegistration`: `wire_domain_registration_migration_finished`
+   - `user`: `wire_user_migration_finished`
+
+   > ⚠️ For user migrations please watch the logs for `Invalid user found,
+   > skipping`. This would be accompanied by an error which is either
+   > `UserHasNoName` or `UserHasNoActivated`. These users are invalid and all
+   > interactions with them were resulting in errors. If these warnings are
+   > ignored, these users will stop existing in the system. If these users are
+   > to be saved, the operator must insert some value as `name` and/or
+   > `activated` and then re-trigger the migration **after** the background
+   > worker finishes migrating the valid users.
 
 3. Cut over reads and writes to PostgreSQL for the selected migration
    setting(s). This configuration must be used from now on for every new
@@ -2253,13 +2267,15 @@ The current settings and their background-worker flags are:
          conversation: postgresql
          conversationCodes: postgresql
          teamFeatures: postgresql
-         domainRegistration: cassandra
+         domainRegistration: postgresql
+         user: postgresql
    background-worker:
      config:
        migrateConversations: false
        migrateConversationCodes: false
        migrateTeamFeatures: false
        migrateDomainRegistration: false
+       migrateUsers: false
    ```
 
 **How to run migrations independently or in batches**

integration/default.nix

@@ -57,6 +57,7 @@
 , optparse-applicative
 , process
 , proto-lens
+, QuickCheck
 , ram
 , random
 , raw-strings-qq
@@ -162,6 +163,7 @@ mkDerivation {
     optparse-applicative
     process
     proto-lens
+    QuickCheck
     ram
     random
     raw-strings-qq

integration/integration.cabal

@@ -181,6 +181,7 @@ library
     Test.Migration.ConversationCodes
     Test.Migration.DomainRegistration
     Test.Migration.TeamFeatures
+    Test.Migration.User
     Test.Migration.Util
     Test.MLS
     Test.MLS.Clients
@@ -301,6 +302,7 @@ library
     , optparse-applicative
     , process
     , proto-lens
+    , QuickCheck               ^>=2.15.0.1
     , ram
     , random
     , raw-strings-qq

integration/test/API/Brig.hs

@@ -156,10 +156,13 @@ getSelfClients u =
 
 -- | https://staging-nginz-https.zinfra.io/v5/api/swagger-ui/#/default/delete_self
 deleteUser :: (HasCallStack, MakesValue user) => user -> App Response
-deleteUser user = do
+deleteUser user = deleteUserWithPassword user (Just defPassword)
+
+deleteUserWithPassword :: (HasCallStack, MakesValue user) => user -> Maybe String -> App Response
+deleteUserWithPassword user mPassword = do
   req <- baseRequest user Brig Versioned "/self"
   submit "DELETE" $
-    req & addJSONObject ["password" .= defPassword]
+    req & addJSONObject ["password" .= mPassword]
 
 -- | https://staging-nginz-https.zinfra.io/v5/api/swagger-ui/#/default/post_clients
 addClient ::
@@ -824,6 +827,15 @@ addBot user providerId serviceId convId = do
       & zType "access"
       & addJSONObject ["provider" .= providerId, "service" .= serviceId]
 
+rmBotSelf :: (HasCallStack, MakesValue domain) => domain -> String -> String -> App Response
+rmBotSelf domain bid cid = do
+  req <- rawBaseRequest domain Brig Versioned $ joinHttpPath ["bot", "self"]
+  submit "DELETE" $
+    req
+      & zType "bot"
+      & addHeader "Z-Bot" bid
+      & addHeader "Z-Conversation" cid
+
 setProperty :: (MakesValue user, ToJSON val) => user -> String -> val -> App Response
 setProperty user propName val = do
   req <- baseRequest user Brig Versioned $ joinHttpPath ["properties", propName]

integration/test/API/Common.hs

@@ -25,6 +25,7 @@ import qualified Data.ByteString as BS
 import Data.Scientific (scientific)
 import qualified Data.Vector as Vector
 import System.Random (randomIO, randomRIO)
+import Test.QuickCheck
 import Testlib.Prelude
 
 -- | please don't use special shell characters like '!' here.  it makes writing shell lines
@@ -33,8 +34,11 @@ defPassword :: String
 defPassword = "hunter2."
 
 randomEmail :: App String
-randomEmail = do
-  u <- randomName
+randomEmail = liftIO $ generate arbitraryEmail
+
+arbitraryEmail :: Gen String
+arbitraryEmail = do
+  u <- arbitraryName
   pure $ u <> "@example.com"
 
 randomDomain :: App String
@@ -52,23 +56,32 @@ randomExternalId = liftIO $ do
     pick = (chars !) <$> randomRIO (Array.bounds chars)
 
 randomName :: App String
-randomName = liftIO $ do
-  n <- randomRIO (8, 15)
+randomName = liftIO $ generate arbitraryName
+
+arbitraryName :: Gen String
+arbitraryName = do
+  n <- chooseInt (8, 15)
   replicateM n pick
   where
     chars = mkArray $ ['A' .. 'Z'] <> ['a' .. 'z'] <> ['0' .. '9']
-    pick = (chars !) <$> randomRIO (Array.bounds chars)
+    pick = (chars !) <$> chooseInt (Array.bounds chars)
 
 randomHandle :: App String
-randomHandle = randomHandleWithRange 50 256
+randomHandle = liftIO $ generate arbitraryHandle
 
 randomHandleWithRange :: Int -> Int -> App String
-randomHandleWithRange min' max' = liftIO $ do
-  n <- randomRIO (min', max')
+randomHandleWithRange min' max' = liftIO $ generate (arbitraryHandleWithRange min' max')
+
+arbitraryHandle :: Gen String
+arbitraryHandle = arbitraryHandleWithRange 50 60
+
+arbitraryHandleWithRange :: Int -> Int -> Gen String
+arbitraryHandleWithRange min' max' = do
+  n <- chooseInt (min', max')
   replicateM n pick
   where
     chars = mkArray $ ['a' .. 'z'] <> ['0' .. '9'] <> "_-."
-    pick = (chars !) <$> randomRIO (Array.bounds chars)
+    pick = (chars !) <$> chooseInt (Array.bounds chars)
 
 randomBytes :: Int -> App ByteString
 randomBytes n = liftIO $ BS.pack <$> replicateM n randomIO
@@ -85,6 +98,14 @@ randomAlphaString n = liftIO $ replicateM n pick
     chars = mkArray $ ['A' .. 'Z'] <> ['a' .. 'z'] <> ['0' .. '9']
     pick = (chars !) <$> randomRIO (Array.bounds chars)
 
+randomPassword :: App String
+randomPassword = liftIO $ generate arbitraryPassword
+
+arbitraryPassword :: Gen String
+arbitraryPassword = do
+  n <- chooseInt (8, 1024)
+  replicateM n arbitraryPrintableChar
+
 randomJSON :: App Value
 randomJSON = do
   let maxThings = 5

integration/test/SetupHelpers.hs

@@ -605,10 +605,14 @@ getCookieWithSamlLogin ::
   App (Maybe String, SAML.SignedAuthnResponse)
 getCookieWithSamlLogin mbZHost domain expectSuccess tid nameId mLabel (iid, (meta, privcreds)) = do
   let idpConfig = SAML.IdPConfig (SAML.IdPId (fromMaybe (error "invalid idp id") (UUID.fromString iid))) meta ()
-  spmeta <- getSPMetadataWithZHost domain mbZHost tid
-  authnreq <- initiateSamlLoginWithZHostAndLabel domain mbZHost mLabel iid
-  let spMetaData = toSPMetaData spmeta.body
-      parsedAuthnReq = parseAuthnReqResp authnreq.body
+  spMetaData <-
+    getSPMetadataWithZHost domain mbZHost tid `bindResponse` \resp -> do
+      resp.status `shouldMatchInt` 200
+      pure $ toSPMetaData resp.body
+  parsedAuthnReq <-
+    initiateSamlLoginWithZHostAndLabel domain mbZHost mLabel iid `bindResponse` \resp -> do
+      resp.status `shouldMatchInt` 200
+      pure $ parseAuthnReqResp resp.body
   authnReqResp <- makeAuthnResponse nameId privcreds idpConfig spMetaData parsedAuthnReq
   mCookie <- finalizeSamlLoginWithZHost domain mbZHost tid authnReqResp `bindResponse` validateLoginResp
   pure (mCookie, authnReqResp)
@@ -653,13 +657,14 @@ makeAuthnResponse nameId privcreds idpConfig spMetaData parsedAuthnReq =
 
 -- | extract an `AuthnRequest` from the html form in the http response from /sso/initiate-login
 parseAuthnReqResp ::
+  (HasCallStack) =>
   ByteString ->
   SAML.AuthnRequest
 parseAuthnReqResp bs = reqBody
   where
     xml :: XML.Document
     xml =
-      fromRight (error "malformed html in response body") $
+      fromRight (error $ "malformed html in response body: \n" <> show bs) $
         XML.parseText XML.def (cs bs)
 
     reqBody :: SAML.AuthnRequest
@@ -855,12 +860,12 @@ createNewIndex = do
     ExitFailure _ -> assertFailure $ prefix <> "failed to create index"
     ExitSuccess -> pure indexName
 
-reindexUsers :: (HasCallStack) => BackendResource -> Int -> App ()
-reindexUsers ber pageSize = do
+reindexUsers :: (HasCallStack) => BackendResource -> ServiceOverrides -> Int -> App ()
+reindexUsers ber serviceOverrides pageSize = do
   testName <- asks (fromMaybe "NoTest" . (.currentTestName))
   let indexName = ber.berElasticsearchIndex
   let prefix = "[reindex-users:" <> indexName <> ":" <> testName <> "] "
-  getBrigConfig <- readAndUpdateConfig (defaultOverrides ber) ber Brig
+  getBrigConfig <- readAndUpdateConfig (defaultOverrides ber <> serviceOverrides) ber Brig
   brigConfig <- liftIO $ getBrigConfig
   esServer <- brigConfig %. "elasticsearch.url" & asString
   esCredentials <- brigConfig %. "elasticsearch.credentials" & asString

integration/test/Test/Migration/Conversation.hs

@@ -99,6 +99,7 @@ testMigrationToPostgresMLS = do
         runPhase 5
   where
     n = 1
+
     createTestConvs :: (HasCallStack) => ClientIdentity -> String -> ClientIdentity -> ClientIdentity -> [ClientIdentity] -> App TestConvList
     createTestConvs creatorC tid melC markC othersC = do
       unmodifiedConvs <- replicateM n $ do