diff --git a/src/actions.test.ts b/src/actions.test.ts new file mode 100644 index 0000000..e760a2b --- /dev/null +++ b/src/actions.test.ts @@ -0,0 +1,271 @@ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import type { Socket } from "bun"; + +import { getFreePortNumber } from "redis-monorepo/packages/test-utils/lib/proxy/redis-proxy.ts"; +import { createApp } from "./app"; +import createMockRedisServer from "./mock-server"; + +// Shrink the migration windows so the suite stays fast. The action code reads +// these at execution time. +process.env.MIGRATION_DELAY_MS = "300"; +process.env.COMPLETION_DELAY_MS = "100"; + +interface ActionStatusResponse { + status: string; + error: unknown; + output: unknown; +} + +describe("Fault Injector action API", () => { + let app: any; + let mockRedisServer: ReturnType; + let listenPort: number; + let targetPort: number; + + const postAction = async (body: unknown): Promise => { + return app.request("/action", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + }; + + const submitAction = async (body: unknown): Promise => { + const res = await postAction(body); + expect(res.status).toBe(200); + const { action_id } = (await res.json()) as { action_id: string }; + expect(action_id).toBeString(); + return action_id; + }; + + const waitForAction = async (actionId: string): Promise => { + const deadline = Date.now() + 10_000; + while (Date.now() < deadline) { + const res = await app.request(`/action/${actionId}`); + expect(res.status).toBe(200); + const action = (await res.json()) as ActionStatusResponse; + if (action.status !== "pending" && action.status !== "running") { + return action; + } + await new Promise((resolve) => setTimeout(resolve, 25)); + } + throw new Error(`Timeout waiting for action ${actionId}`); + }; + + const runAction = async (body: unknown): Promise => { + return waitForAction(await submitAction(body)); + }; + + const getNodeIds = async (): Promise => { + const res = await app.request("/nodes"); + return (await res.json()).ids; + }; + + const resetCluster = async () => { + const result = await runAction({ type: "reset_cluster", parameters: {} }); + expect(result.status).toBe("success"); + }; + + beforeAll(async () => { + listenPort = await getFreePortNumber(); + targetPort = await getFreePortNumber(); + + mockRedisServer = createMockRedisServer(targetPort); + + const appInstance = createApp({ + listenPort: [listenPort], + listenHost: "127.0.0.1", + targetHost: "127.0.0.1", + targetPort: targetPort, + timeout: 30000, + enableLogging: false, + apiPort: 3002, + }); + app = appInstance.app; + + await new Promise((resolve) => setTimeout(resolve, 200)); + }); + + afterAll(async () => { + const res = await app.request("/nodes"); + const { ids } = await res.json(); + for (const id of ids) { + await app.request(`/nodes/${encodeURIComponent(id)}`, { method: "DELETE" }); + } + mockRedisServer?.stop(true); + }); + + test("POST /action rejects unknown action types", async () => { + const res = await postAction({ type: "bogus", parameters: {} }); + expect(res.status).toBe(400); + }); + + test("GET /slot-migrate returns FI-shaped triggers", async () => { + const res = await app.request("/slot-migrate?effect=remove"); + expect(res.status).toBe(200); + + const body = await res.json(); + expect(body.effect).toBe("remove"); + expect(body.cluster.index).toBe(0); + expect(body.cluster.nodes).toBeNumber(); + + const names = body.triggers.map((trigger: { name: string }) => trigger.name); + expect(names).toEqual(["migrate", "maintenance_mode", "failover"]); + + for (const trigger of body.triggers) { + expect(trigger.description).toBeString(); + expect(trigger.requirements.length).toBeGreaterThan(0); + for (const requirement of trigger.requirements) { + expect(requirement.dbconfig.name).toBeString(); + expect(requirement.dbconfig.name).toContain("sm-remove-"); + expect(requirement.dbconfig.shards_count).toBeNumber(); + expect(requirement.cluster.min_nodes).toBe(3); + expect(requirement.description).toBeString(); + } + } + + // The --db=ext-hostname CLI filter of the scenario tests matches on this + const dbNames = body.triggers.flatMap( + (trigger: { requirements: { dbconfig: { name: string } }[] }) => + trigger.requirements.map((requirement) => requirement.dbconfig.name), + ); + expect(dbNames.some((name: string) => name.includes("ext-ip"))).toBe(true); + expect(dbNames.some((name: string) => name.includes("ext-hostname"))).toBe(true); + }); + + test("GET /slot-migrate requires an effect", async () => { + const res = await app.request("/slot-migrate"); + expect(res.status).toBe(400); + }); + + test("create_database sizes the cluster and returns connection info", async () => { + const result = await runAction({ + type: "create_database", + parameters: { + cluster_index: 0, + database_config: { name: "sm-remove-migrate-ext-ip", shards_count: 3 }, + }, + }); + + expect(result.status).toBe("success"); + const output = result.output as { + bdb_id: number; + username: string; + password: string; + tls: boolean; + raw_endpoints: { dns_name: string; port: number }[]; + }; + expect(output.bdb_id).toBeNumber(); + expect(output.tls).toBe(false); + expect(output.raw_endpoints.length).toBe(3); + expect(output.raw_endpoints[0]?.dns_name).toBe("127.0.0.1"); + expect(output.raw_endpoints[0]?.port).toBeNumber(); + + expect((await getNodeIds()).length).toBe(3); + }); + + test("reset_cluster restores the initial topology", async () => { + await resetCluster(); + expect((await getNodeIds()).length).toBe(1); + }); + + test("GET /action lists submitted actions", async () => { + const actionId = await submitAction({ type: "wait", parameters: {} }); + await waitForAction(actionId); + + const res = await app.request("/action"); + expect(res.status).toBe(200); + const { actions } = await res.json(); + const entry = actions.find((action: { job_id: string }) => action.job_id === actionId); + expect(entry).toBeDefined(); + expect(entry.action_type).toBe("wait"); + expect(entry.submitted_at).toBeString(); + }); + + test("slot_migrate rejects an invalid effect", async () => { + const result = await runAction({ + type: "slot_migrate", + parameters: { effect: "explode", cluster_index: 0 }, + }); + expect(result.status).toBe("failed"); + expect(String(result.error)).toContain("Invalid effect"); + }); + + test("slot_migrate remove notifies all clients and new connections", async () => { + await resetCluster(); + const createResult = await runAction({ + type: "create_database", + parameters: { + cluster_index: 0, + database_config: { name: "sm-remove-migrate-ext-ip", shards_count: 3 }, + }, + }); + const { raw_endpoints } = createResult.output as { + raw_endpoints: { dns_name: string; port: number }[]; + }; + expect(raw_endpoints.length).toBe(3); + + const connectAndCollect = async (port: number) => { + const chunks: Buffer[] = []; + let socket: Socket | undefined; + await new Promise((resolve, reject) => { + Bun.connect({ + hostname: "127.0.0.1", + port, + socket: { + open(openedSocket) { + socket = openedSocket; + resolve(); + }, + data(_socket, data) { + chunks.push(Buffer.from(data)); + }, + error(_socket, error) { + reject(error); + }, + close() {}, + }, + }); + }); + return { + received: () => Buffer.concat(chunks).toString(), + close: () => socket?.end(), + }; + }; + + const clients = await Promise.all( + raw_endpoints.map((endpoint) => connectAndCollect(endpoint.port)), + ); + // Let the proxies register the connections + await new Promise((resolve) => setTimeout(resolve, 100)); + + const actionId = await submitAction({ + type: "slot_migrate", + parameters: { effect: "remove", cluster_index: 0, trigger: "migrate", bdb_id: "1" }, + }); + + // SMIGRATING is broadcast right away; the migration window is 300ms + await new Promise((resolve) => setTimeout(resolve, 100)); + for (const client of clients) { + expect(client.received()).toContain("SMIGRATING"); + } + + // A connection opened during the migration window gets SMIGRATING too + const lateClient = await connectAndCollect(raw_endpoints[0]?.port as number); + await new Promise((resolve) => setTimeout(resolve, 150)); + expect(lateClient.received()).toContain("SMIGRATING"); + + const result = await waitForAction(actionId); + expect(result.status).toBe("success"); + + // One node was removed and the survivors got SMIGRATED + expect((await getNodeIds()).length).toBe(2); + const migrated = clients.filter((client) => client.received().includes("SMIGRATED")); + expect(migrated.length).toBeGreaterThan(0); + + lateClient.close(); + for (const client of clients) { + client.close(); + } + }); +}); diff --git a/src/actions/index.ts b/src/actions/index.ts new file mode 100644 index 0000000..7a61485 --- /dev/null +++ b/src/actions/index.ts @@ -0,0 +1,460 @@ +import { + type ProxyConfig, + RedisProxy, +} from "redis-monorepo/packages/test-utils/lib/proxy/redis-proxy"; +import applyDefaultInterceptors from "../default_interceptors/index"; +import type ProxyStore from "../proxy-store"; +import { makeId } from "../proxy-store"; +import { + addNode, + buildSMigratedNotification, + buildSMigratingNotification, + createCustomClusterSlotsInterceptor, + findNextAvailablePort, + getSlotRangesForProxy, + pickRandom, + sendToAllClients, +} from "../scenarios/helpers"; +import { getNextSequenceId } from "../scenarios/sequence-gen"; +import type { ActionType, ExtendedProxyConfig } from "../util"; + +// Effect types matching Python MigrateEffect enum +export type SlotMigrateEffect = "remove-add" | "remove" | "add" | "slot-shuffle"; + +export interface SlotMigrateParams { + effect: SlotMigrateEffect; + variant?: string; + source_node?: number; + target_node?: number; +} + +export interface ActionExecutionResult { + status: "success" | "failed"; + error?: string | null; + output?: unknown; +} + +const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +// The window between SMIGRATING and SMIGRATED, and the settle time before a +// removed node is stopped. Overridable so tests do not wait several seconds. +const migrationDelayMs = () => Number(process.env.MIGRATION_DELAY_MS ?? 5000); +const completionDelayMs = () => Number(process.env.COMPLETION_DELAY_MS ?? 2000); + +// How long to wait after a client connects before pushing SMIGRATING to it, +// so the push lands after the client finished its HELLO handshake. +const NEW_CONNECTION_PUSH_DELAY_MS = 25; + +/** + * While a migration is active, push the SMIGRATING notification to every + * client that connects, matching how a real cluster treats connections + * opened while a migration is in progress. + * Returns a function that stops the notifications. + */ +function pushToNewConnections(proxyStore: ProxyStore, buffer: Buffer): () => void { + const subscriptions = proxyStore.proxies.map((proxy) => { + const listener = (connection: { id: string }) => { + setTimeout(() => proxy.sendToClient(connection.id, buffer), NEW_CONNECTION_PUSH_DELAY_MS); + }; + proxy.on("connection", listener); + return () => proxy.off("connection", listener); + }); + return () => { + for (const unsubscribe of subscriptions) unsubscribe(); + }; +} + +async function startNode(proxyStore: ProxyStore, config: ProxyConfig): Promise { + const proxy = new RedisProxy(config); + // Without a listener, the 'error' the proxy emits alongside a failed + // start() escapes the EventEmitter and crashes the process. + proxy.on("error", (error: Error) => console.error("[proxy]", error.message)); + await proxy.start(); + proxyStore.add(makeId(config.targetHost, config.targetPort, config.listenPort), proxy); + return proxy; +} + +/** Next free listen port, never colliding with the backend target port. */ +function nextListenPort(proxyStore: ProxyStore, config: ExtendedProxyConfig): number { + let port = + proxyStore.proxies.length > 0 + ? findNextAvailablePort(proxyStore.proxies) + : Math.max(...config.listenPort); + while (port === config.targetPort) port++; + return port; +} + +async function removeNode(proxyStore: ProxyStore, proxy: RedisProxy): Promise { + const { targetHost, targetPort, listenPort } = proxy.config; + await proxyStore.delete(makeId(targetHost, targetPort, listenPort)); +} + +function refreshClusterSlots(proxyStore: ProxyStore): void { + const interceptor = createCustomClusterSlotsInterceptor(proxyStore.proxies); + for (const proxy of proxyStore.proxies) { + proxy.addGlobalInterceptor(interceptor); + } +} + +/** + * Execute an action based on its type and parameters + */ +export async function executeAction( + actionType: ActionType, + parameters: Record, + proxyStore: ProxyStore, + config: ExtendedProxyConfig, +): Promise { + switch (actionType) { + case "slot_migrate": + return executeSlotMigrate(parameters as unknown as SlotMigrateParams, proxyStore, config); + case "reset_cluster": + return executeResetCluster(proxyStore, config); + case "create_database": + return executeCreateDatabase(parameters, proxyStore, config); + default: + return { status: "success" }; + } +} + +/** + * Restore the initial proxy topology and drop interceptors added by earlier + * actions. Test harnesses call this before every test. + */ +async function executeResetCluster( + proxyStore: ProxyStore, + config: ExtendedProxyConfig, +): Promise { + try { + for (const id of proxyStore.nodeIds) { + await proxyStore.delete(id); + } + for (const port of config.listenPort) { + await startNode(proxyStore, { ...config, listenPort: port }); + } + if (config.defaultInterceptors) { + applyDefaultInterceptors(config.defaultInterceptors, proxyStore); + } + return { status: "success" }; + } catch (error) { + return { status: "failed", error: error instanceof Error ? error.message : String(error) }; + } +} + +/** + * Size the proxy cluster to the requested shards_count and return connection + * info in the shape Fault Injector clients expect: they read + * raw_endpoints[0], username, password, tls and bdb_id from the output. + */ +async function executeCreateDatabase( + parameters: Record, + proxyStore: ProxyStore, + config: ExtendedProxyConfig, +): Promise { + try { + const databaseConfig = (parameters.database_config ?? {}) as Record; + const shardsCount = + typeof databaseConfig.shards_count === "number" && databaseConfig.shards_count > 0 + ? databaseConfig.shards_count + : Math.max(proxyStore.proxies.length, 1); + + while (proxyStore.proxies.length > shardsCount) { + const proxy = proxyStore.proxies.at(-1); + if (!proxy) break; + await removeNode(proxyStore, proxy); + } + while (proxyStore.proxies.length < shardsCount) { + await startNode(proxyStore, { ...config, listenPort: nextListenPort(proxyStore, config) }); + } + + refreshClusterSlots(proxyStore); + + return { + status: "success", + output: { + bdb_id: 1, + username: "", + password: "", + tls: false, + raw_endpoints: proxyStore.proxies.map((proxy) => ({ + dns_name: proxy.config.listenHost, + port: proxy.config.listenPort, + })), + }, + }; + } catch (error) { + return { status: "failed", error: error instanceof Error ? error.message : String(error) }; + } +} + +/** + * Execute slot_migrate action - handles all scenario effects + */ +async function executeSlotMigrate( + params: SlotMigrateParams, + proxyStore: ProxyStore, + config: ExtendedProxyConfig, +): Promise { + const { effect } = params; + + if (!effect) { + return { status: "failed", error: "Missing required parameter: effect" }; + } + + const validEffects: SlotMigrateEffect[] = ["remove-add", "remove", "add", "slot-shuffle"]; + if (!validEffects.includes(effect)) { + return { + status: "failed", + error: `Invalid effect: ${effect}. Must be one of: ${validEffects.join(", ")}`, + }; + } + + try { + switch (effect) { + case "remove-add": + await executeRemoveAddEffect(proxyStore, config); + break; + case "remove": + await executeRemoveEffect(proxyStore, config); + break; + case "add": + await executeAddEffect(proxyStore, config); + break; + case "slot-shuffle": + await executeSlotShuffleEffect(proxyStore, config); + break; + } + return { status: "success" }; + } catch (error) { + return { status: "failed", error: error instanceof Error ? error.message : String(error) }; + } +} + +async function executeRemoveAddEffect( + proxyStore: ProxyStore, + config: ExtendedProxyConfig, +): Promise { + const allProxies = proxyStore.proxies; + if (allProxies.length === 0) { + throw new Error("No proxies available to select from"); + } + + const proxyToBeRemoved = pickRandom(allProxies); + if (!proxyToBeRemoved) { + throw new Error("Failed to select a random proxy"); + } + + const slotRanges = getSlotRangesForProxy(proxyToBeRemoved, allProxies); + const newPort = nextListenPort(proxyStore, config); + const newProxyConfig: ProxyConfig = { ...config, listenPort: newPort }; + const { proxy: newProxy } = addNode(proxyStore, newProxyConfig); + + const proxiesForClusterSlots = allProxies.filter((p) => p !== proxyToBeRemoved).concat(newProxy); + const clusterSlotsInterceptor = createCustomClusterSlotsInterceptor(proxiesForClusterSlots); + + for (const proxy of proxyStore.proxies) { + proxy.addGlobalInterceptor(clusterSlotsInterceptor); + } + + const sMigratingBuffer = buildSMigratingNotification(slotRanges, getNextSequenceId()); + sendToAllClients(proxyStore, sMigratingBuffer); + const stopNotifying = pushToNewConnections(proxyStore, sMigratingBuffer); + + await delay(migrationDelayMs()); + + const sMigratedBuffer = buildSMigratedNotification( + [ + { + targetNode: { host: newProxy.config.listenHost, port: newProxy.config.listenPort }, + slotRanges, + }, + ], + getNextSequenceId(), + ); + sendToAllClients(proxyStore, sMigratedBuffer); + stopNotifying(); + + await delay(completionDelayMs()); + + await removeNode(proxyStore, proxyToBeRemoved); +} + +async function executeRemoveEffect( + proxyStore: ProxyStore, + _config: ExtendedProxyConfig, +): Promise { + const allProxies = proxyStore.proxies; + if (allProxies.length === 0) { + throw new Error("No proxies available to select from"); + } + if (allProxies.length === 1) { + throw new Error("Cannot remove the last remaining node"); + } + + const proxyToBeRemoved = pickRandom(allProxies); + if (!proxyToBeRemoved) { + throw new Error("Failed to select a random proxy"); + } + + const removedNodeSlotRanges = getSlotRangesForProxy(proxyToBeRemoved, allProxies); + const remainingProxies = allProxies.filter((p) => p !== proxyToBeRemoved); + + const newSlotDistribution = remainingProxies.map((proxy) => ({ + proxy, + slotRanges: getSlotRangesForProxy(proxy, remainingProxies), + })); + + const clusterSlotsInterceptor = createCustomClusterSlotsInterceptor(remainingProxies); + for (const proxy of proxyStore.proxies) { + proxy.addGlobalInterceptor(clusterSlotsInterceptor); + } + + const sMigratingBuffer = buildSMigratingNotification(removedNodeSlotRanges, getNextSequenceId()); + sendToAllClients(proxyStore, sMigratingBuffer); + const stopNotifying = pushToNewConnections(proxyStore, sMigratingBuffer); + + await delay(migrationDelayMs()); + + const migratedSlots = newSlotDistribution.map(({ proxy, slotRanges }) => ({ + targetNode: { host: proxy.config.listenHost, port: proxy.config.listenPort }, + slotRanges, + })); + const sMigratedBuffer = buildSMigratedNotification(migratedSlots, getNextSequenceId()); + sendToAllClients(proxyStore, sMigratedBuffer); + stopNotifying(); + + await delay(completionDelayMs()); + + await removeNode(proxyStore, proxyToBeRemoved); +} + +async function executeAddEffect( + proxyStore: ProxyStore, + config: ExtendedProxyConfig, +): Promise { + const allProxies = proxyStore.proxies; + if (allProxies.length === 0) { + throw new Error("No proxies available"); + } + + const oldSlotDistribution = allProxies.map((proxy) => ({ + proxy, + slotRanges: getSlotRangesForProxy(proxy, allProxies), + })); + + const newPort = nextListenPort(proxyStore, config); + const newProxyConfig: ProxyConfig = { ...config, listenPort: newPort }; + const { proxy: newProxy } = addNode(proxyStore, newProxyConfig); + + const allProxiesWithNew = [...allProxies, newProxy]; + const clusterSlotsInterceptor = createCustomClusterSlotsInterceptor(allProxiesWithNew); + + for (const proxy of proxyStore.proxies) { + proxy.addGlobalInterceptor(clusterSlotsInterceptor); + } + + let sMigratingBuffer: Buffer = Buffer.alloc(0); + for (const { proxy, slotRanges } of oldSlotDistribution) { + sMigratingBuffer = buildSMigratingNotification(slotRanges, getNextSequenceId()); + proxy.sendToAllClients(sMigratingBuffer); + } + const stopNotifying = pushToNewConnections(proxyStore, sMigratingBuffer); + + await delay(migrationDelayMs()); + + const newNodeSlotRanges = getSlotRangesForProxy(newProxy, allProxiesWithNew); + for (const { proxy } of oldSlotDistribution) { + const sMigratedBuffer = buildSMigratedNotification( + [ + { + targetNode: { host: newProxy.config.listenHost, port: newProxy.config.listenPort }, + slotRanges: newNodeSlotRanges, + }, + ], + getNextSequenceId(), + ); + proxy.sendToAllClients(sMigratedBuffer); + } + stopNotifying(); +} + +async function executeSlotShuffleEffect( + proxyStore: ProxyStore, + _config: ExtendedProxyConfig, +): Promise { + const allProxies = proxyStore.proxies; + if (allProxies.length === 0) { + throw new Error("No proxies available"); + } + if (allProxies.length === 1) { + throw new Error("Cannot shuffle slots with only one node"); + } + + const oldSlotDistribution = allProxies.map((proxy) => ({ + proxy, + slotRanges: getSlotRangesForProxy(proxy, allProxies), + })); + + // Fisher-Yates shuffle + const shuffledProxies = [...allProxies]; + for (let i = shuffledProxies.length - 1; i > 0; i--) { + const j = Math.floor(Math.random() * (i + 1)); + const temp = shuffledProxies[i]; + const jProxy = shuffledProxies[j]; + if (temp && jProxy) { + shuffledProxies[i] = jProxy; + shuffledProxies[j] = temp; + } + } + + const newSlotDistribution = shuffledProxies.map((proxy, index) => { + const slotLength = Math.floor(16384 / shuffledProxies.length); + const from = index * slotLength; + const to = index === shuffledProxies.length - 1 ? 16383 : from + slotLength - 1; + return { proxy, slotRanges: `${from}-${to}` }; + }); + + const customShuffledInterceptor = { + name: "cluster-simulation-interceptor", + fn: async ( + data: Buffer, + next: (data: Buffer) => Promise, + state: { invokeCount: number; matchCount: number }, + ) => { + state.invokeCount++; + if (data.toString().toLowerCase() !== "*2\r\n$7\r\ncluster\r\n$5\r\nslots\r\n") { + return next(data); + } + state.matchCount++; + const mapping = newSlotDistribution.map(({ proxy, slotRanges }) => { + const [from, to] = slotRanges.split("-").map(Number); + const id = `proxy-id-${proxy.config.listenPort}`; + return `*3\r\n:${from}\r\n:${to}\r\n*3\r\n$${proxy.config.listenHost.length}\r\n${proxy.config.listenHost}\r\n:${proxy.config.listenPort}\r\n$${id.length}\r\n${id}\r\n`; + }); + return Buffer.from(`*${newSlotDistribution.length}\r\n${mapping.join("")}`); + }, + }; + + for (const proxy of proxyStore.proxies) { + proxy.addGlobalInterceptor(customShuffledInterceptor); + } + + let sMigratingBuffer: Buffer = Buffer.alloc(0); + for (const { proxy, slotRanges } of oldSlotDistribution) { + sMigratingBuffer = buildSMigratingNotification(slotRanges, getNextSequenceId()); + proxy.sendToAllClients(sMigratingBuffer); + } + const stopNotifying = pushToNewConnections(proxyStore, sMigratingBuffer); + + await delay(migrationDelayMs()); + + for (const { proxy: sourceProxy } of oldSlotDistribution) { + const migratedSlots = newSlotDistribution.map(({ proxy, slotRanges }) => ({ + targetNode: { host: proxy.config.listenHost, port: proxy.config.listenPort }, + slotRanges, + })); + const sMigratedBuffer = buildSMigratedNotification(migratedSlots, getNextSequenceId()); + sourceProxy.sendToAllClients(sMigratedBuffer); + } + stopNotifying(); +} diff --git a/src/actions/triggers.ts b/src/actions/triggers.ts new file mode 100644 index 0000000..eba465f --- /dev/null +++ b/src/actions/triggers.ts @@ -0,0 +1,118 @@ +import type { ActionTrigger, ActionTriggerRequirement, SlotMigrateEffect } from "../util"; + +// Mirrors re_fault_injector TRIGGER_DEFINITIONS so clients can select +// triggers by their real names (migrate, maintenance_mode, failover). +const TRIGGER_DEFINITIONS: Record = { + "remove-add": [ + { + name: "migrate", + description: "Use rladmin migrate to move all shards from source node to empty node", + }, + { + name: "maintenance_mode", + description: "Put source node in maintenance mode, shards auto-migrate to other nodes", + }, + { + name: "failover", + description: "Trigger failover to swap master/replica roles (requires replication)", + }, + ], + remove: [ + { + name: "migrate", + description: "Use rladmin migrate to move all shards from source node to existing node", + }, + { + name: "maintenance_mode", + description: "Put source node in maintenance mode, shards auto-migrate to other nodes", + }, + { + name: "failover", + description: "Trigger failover to swap master/replica roles (requires replication)", + }, + ], + add: [ + { name: "migrate", description: "Use rladmin migrate to move one shard to empty node" }, + { + name: "failover", + description: "Trigger failover to swap master/replica roles (requires replication)", + }, + ], + "slot-shuffle": [ + { + name: "migrate", + description: "Use rladmin migrate to move one shard between existing nodes", + }, + { + name: "failover", + description: "Trigger failover to swap master/replica roles (requires replication)", + }, + ], +}; + +// The two default (external) OSS Cluster API combinations the FI returns. +const IP_TYPES = [ + { ipType: "external", endpointType: "ip", suffix: "ext-ip" }, + { ipType: "external", endpointType: "hostname", suffix: "ext-hostname" }, +]; + +const MIN_NODES = 3; +const BASE_PORT = 13000; + +// Mirrors re_fault_injector _calculate_shards_count. For the proxy, +// shards_count doubles as the number of proxy nodes create_database spins up. +function calculateShardsCount(effect: SlotMigrateEffect, trigger: string, nodeCount: number) { + switch (effect) { + case "remove-add": + return trigger === "failover" ? 1 : nodeCount - 1; + case "remove": + return nodeCount; + case "add": + return trigger === "failover" ? 2 : nodeCount; + case "slot-shuffle": + return nodeCount * 2; + } +} + +function calculatePlacement(effect: SlotMigrateEffect, trigger: string) { + if (effect === "add") return "dense"; + if (effect === "remove-add" && trigger === "maintenance_mode") return "dense"; + return "sparse"; +} + +export function generateTriggersForEffect( + effect: SlotMigrateEffect, + nodeCount: number, +): ActionTrigger[] { + // The FI derives shards_count from the RE cluster size and needs >= 3 nodes + // for quorum. The proxy has no quorum and can add nodes freely, so size the + // dbconfigs as if the cluster had at least MIN_NODES. + const effectiveNodes = Math.max(nodeCount, MIN_NODES); + + return TRIGGER_DEFINITIONS[effect].map((trigger) => ({ + name: trigger.name, + description: trigger.description, + requirements: IP_TYPES.map(({ ipType, endpointType, suffix }, index) => { + const requirement: ActionTriggerRequirement = { + dbconfig: { + name: `sm-${effect}-${trigger.name.replace(/_/g, "-")}-${suffix}`, + port: BASE_PORT + index, + memory_size: 134217728, + eviction_policy: "volatile-lru", + sharding: true, + oss_cluster: true, + proxy_policy: "all-master-shards", + shards_count: calculateShardsCount(effect, trigger.name, effectiveNodes), + shards_placement: calculatePlacement(effect, trigger.name), + replication: trigger.name === "failover", + oss_cluster_api_preferred_ip_type: ipType, + oss_cluster_api_preferred_endpoint_type: endpointType, + }, + cluster: { min_nodes: MIN_NODES, actual_nodes: nodeCount }, + oss_cluster_api: { ip_type: ipType, endpoint_type: endpointType }, + description: `Config (${ipType}/${endpointType})`, + }; + return requirement; + }), + })); +} diff --git a/src/app.ts b/src/app.ts index e6d751f..00851e1 100644 --- a/src/app.ts +++ b/src/app.ts @@ -10,18 +10,24 @@ import { RedisProxy, type SendResult, } from "redis-monorepo/packages/test-utils/lib/proxy/redis-proxy.ts"; +import { executeAction } from "./actions/index.ts"; +import { generateTriggersForEffect } from "./actions/triggers.ts"; import applyDefaultInterceptors from "./default_interceptors/index.ts"; import ProxyStore, { makeId } from "./proxy-store.ts"; import { + type ActionRecord, + actionIdParamSchema, + actionRequestSchema, connectionIdsQuerySchema, type ExtendedProxyConfig, encodingSchema, getConfig, interceptorSchema, + type ListActionTriggersResponse, paramSchema, parseBuffer, proxyConfigSchema, - scenarioSchema, + slotMigrateEffectSchema, } from "./util.ts"; const startNewProxy = (config: ProxyConfig) => { @@ -45,6 +51,30 @@ export function createApp(testConfig?: ExtendedProxyConfig) { config.defaultInterceptors && applyDefaultInterceptors(config.defaultInterceptors, proxyStore); + // Simulate the endpoints being offline: drop every client connection and + // stop accepting new ones until rejecting is stopped again. + let rejectingTraffic = false; + + app.post("/reject-traffic/start", async (c) => { + if (!rejectingTraffic) { + rejectingTraffic = true; + for (const proxy of proxyStore.proxies) { + await proxy.stop(); + } + } + return c.json({ success: true, rejecting: rejectingTraffic }); + }); + + app.post("/reject-traffic/stop", async (c) => { + if (rejectingTraffic) { + rejectingTraffic = false; + for (const proxy of proxyStore.proxies) { + await proxy.start(); + } + } + return c.json({ success: true, rejecting: rejectingTraffic }); + }); + app.post("/nodes", zValidator("json", proxyConfigSchema), async (c) => { const data = await c.req.json(); const cfg: ProxyConfig = { ...config, ...data }; @@ -146,33 +176,6 @@ export function createApp(testConfig?: ExtendedProxyConfig) { return c.json({ success, connectionId }); }); - app.post("/scenarios", zValidator("json", scenarioSchema), async (c) => { - const { responses, encoding } = c.req.valid("json"); - - const responsesBuffers = responses.map((response) => parseBuffer(response, encoding)); - let currentIndex = 0; - - const scenarioInterceptor: InterceptorDescription = { - name: "scenario-interceptor", - fn: async (data: Buffer, next: Next, state: InterceptorState): Promise => { - state.invokeCount++; - if (currentIndex < responsesBuffers.length) { - state.matchCount++; - const response = responsesBuffers[currentIndex] as Buffer; - currentIndex++; - return response; - } - return await next(data); - }, - }; - - for (const proxy of proxyStore.proxies) { - proxy.addGlobalInterceptor(scenarioInterceptor); - } - - return c.json({ success: true, totalResponses: responses.length }); - }); - app.post("/interceptors", zValidator("json", interceptorSchema), async (c) => { const { name, match, response, encoding } = c.req.valid("json"); @@ -198,5 +201,86 @@ export function createApp(testConfig?: ExtendedProxyConfig) { return c.json({ success: true, name }); }); + // In-memory action storage + const actionStore = new Map(); + + // Generate unique action ID + const generateActionId = (): string => { + return `action-${Date.now()}-${Math.random().toString(36).substring(2, 9)}`; + }; + + // POST /action - Submit an action + app.post("/action", zValidator("json", actionRequestSchema), async (c) => { + const { type, parameters } = c.req.valid("json"); + + const actionId = generateActionId(); + const actionRecord: ActionRecord = { + id: actionId, + type, + parameters, + status: "pending", + submittedAt: new Date(), + error: null, + output: null, + }; + + actionStore.set(actionId, actionRecord); + + // Execute the action asynchronously + actionRecord.status = "running"; + executeAction(type, parameters, proxyStore, config) + .then((result) => { + actionRecord.status = result.status; + actionRecord.output = result.output ?? "Done"; + actionRecord.error = result.error ?? null; + }) + .catch((error) => { + actionRecord.status = "failed"; + actionRecord.error = error instanceof Error ? error.message : String(error); + }); + + return c.json({ action_id: actionId }); + }); + + // GET /action/:action_id - Get action status + app.get("/action/:action_id", zValidator("param", actionIdParamSchema), (c) => { + const { action_id } = c.req.valid("param"); + + const action = actionStore.get(action_id); + if (!action) { + return c.json({ error: "Action not found" }, 404); + } + + return c.json({ + status: action.status, + error: action.error, + output: action.output, + }); + }); + + // GET /action - List all submitted actions + app.get("/action", (c) => { + const actions = Array.from(actionStore.values()).map((action) => ({ + job_id: action.id, + action_type: action.type, + status: action.status, + submitted_at: action.submittedAt.toISOString(), + })); + return c.json({ actions }); + }); + + // GET /slot-migrate - List action triggers for an effect + app.get("/slot-migrate", zValidator("query", slotMigrateEffectSchema), (c) => { + const { effect } = c.req.valid("query"); + + const response: ListActionTriggersResponse = { + effect, + cluster: { index: 0, nodes: proxyStore.nodeIds.length }, + triggers: generateTriggersForEffect(effect, proxyStore.nodeIds.length), + }; + + return c.json(response); + }); + return { app, proxy: proxyStore.proxies[0] as RedisProxy, config }; } diff --git a/src/reject-traffic.test.ts b/src/reject-traffic.test.ts new file mode 100644 index 0000000..a1bac6e --- /dev/null +++ b/src/reject-traffic.test.ts @@ -0,0 +1,117 @@ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import type { Socket } from "bun"; + +import { getFreePortNumber } from "redis-monorepo/packages/test-utils/lib/proxy/redis-proxy.ts"; +import { createApp } from "./app"; +import createMockRedisServer from "./mock-server"; + +describe("Reject traffic", () => { + let app: any; + let mockRedisServer: ReturnType; + let listenPort: number; + let targetPort: number; + + const connectClient = async (port: number) => { + let socket: Socket | undefined; + let closed = false; + let onClose = () => {}; + await new Promise((resolve, reject) => { + Bun.connect({ + hostname: "127.0.0.1", + port, + socket: { + open(openedSocket) { + socket = openedSocket; + resolve(); + }, + data() {}, + error(_socket, error) { + reject(error); + }, + close() { + closed = true; + onClose(); + }, + }, + }).catch(reject); + }); + return { + isClosed: () => closed, + waitForClose: () => + new Promise((resolve, reject) => { + if (closed) return resolve(); + onClose = resolve; + setTimeout(() => reject(new Error("connection was not closed")), 2000); + }), + close: () => socket?.end(), + }; + }; + + beforeAll(async () => { + listenPort = await getFreePortNumber(); + targetPort = await getFreePortNumber(); + + mockRedisServer = createMockRedisServer(targetPort); + + const appInstance = createApp({ + listenPort: [listenPort], + listenHost: "127.0.0.1", + targetHost: "127.0.0.1", + targetPort: targetPort, + timeout: 30000, + enableLogging: false, + apiPort: 3003, + }); + app = appInstance.app; + + await new Promise((resolve) => setTimeout(resolve, 200)); + }); + + afterAll(async () => { + const res = await app.request("/nodes"); + const { ids } = await res.json(); + for (const id of ids) { + await app.request(`/nodes/${encodeURIComponent(id)}`, { method: "DELETE" }); + } + mockRedisServer?.stop(true); + }); + + test("start drops connections and refuses new ones, stop restores service", async () => { + const client = await connectClient(listenPort); + expect(client.isClosed()).toBe(false); + + const startRes = await app.request("/reject-traffic/start", { method: "POST" }); + expect(startRes.status).toBe(200); + expect(await startRes.json()).toEqual({ success: true, rejecting: true }); + + // The existing connection is dropped + await client.waitForClose(); + + // New connections are refused while rejecting + await expect(connectClient(listenPort)).rejects.toThrow(); + + const stopRes = await app.request("/reject-traffic/stop", { method: "POST" }); + expect(stopRes.status).toBe(200); + expect(await stopRes.json()).toEqual({ success: true, rejecting: false }); + + // Service is back: new connections are accepted again + const revivedClient = await connectClient(listenPort); + expect(revivedClient.isClosed()).toBe(false); + revivedClient.close(); + }); + + test("start and stop are idempotent", async () => { + for (const _ of [1, 2]) { + const res = await app.request("/reject-traffic/start", { method: "POST" }); + expect((await res.json()).rejecting).toBe(true); + } + for (const _ of [1, 2]) { + const res = await app.request("/reject-traffic/stop", { method: "POST" }); + expect((await res.json()).rejecting).toBe(false); + } + + const client = await connectClient(listenPort); + expect(client.isClosed()).toBe(false); + client.close(); + }); +}); diff --git a/src/scenarios.test.ts b/src/scenarios.test.ts deleted file mode 100644 index 1b57cbb..0000000 --- a/src/scenarios.test.ts +++ /dev/null @@ -1,153 +0,0 @@ -import { afterAll, beforeAll, describe, expect, test } from "bun:test"; -import type { SimpleStringReply } from "@redis/client/dist/lib/RESP/types"; -import { createClient } from "redis"; - -import { getFreePortNumber } from "redis-monorepo/packages/test-utils/lib/proxy/redis-proxy.ts"; -import { createApp } from "./app"; -import createMockRedisServer from "./mock-server"; - -describe("POST /scenarios", () => { - let app: any; - let proxy: any; - let mockRedisServer: any; - let targetPort: number; - - beforeAll(async () => { - const freePort = await getFreePortNumber(); - targetPort = await getFreePortNumber(); - - mockRedisServer = createMockRedisServer(targetPort); - - const testConfig = { - listenPort: [freePort], - listenHost: "127.0.0.1", - targetHost: "127.0.0.1", - targetPort: targetPort, - timeout: 30000, - enableLogging: true, - apiPort: 3001, - }; - - const appInstance = createApp(testConfig); - app = appInstance.app; - proxy = appInstance.proxy; - - await new Promise((resolve) => setTimeout(resolve, 200)); - }); - - afterAll(async () => { - if (proxy) { - await proxy.stop(); - } - if (mockRedisServer) { - mockRedisServer?.stop(true); - } - }); - - test("POST /scenarios with invalid data", async () => { - const res = await app.request("/scenarios", { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({}), - }); - - expect(res.status).toBe(400); - }); - - test("POST /scenarios with empty responses", async () => { - const res = await app.request("/scenarios", { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ responses: [] }), - }); - - expect(res.status).toBe(400); - }); - - test("POST /scenarios with raw encoding", async () => { - const res = await app.request("/scenarios", { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ - responses: ["+FIRST\r\n", "+SECOND\r\n", "+THIRD\r\n"], - encoding: "raw", - }), - }); - - expect(res.status).toBe(200); - const result = await res.json(); - expect(result.success).toBe(true); - expect(result.totalResponses).toBe(3); - }); - - test("POST /scenarios with base64 encoding", async () => { - const response1 = Buffer.from("+RESPONSE1\r\n").toString("base64"); - const response2 = Buffer.from("+RESPONSE2\r\n").toString("base64"); - - const res = await app.request("/scenarios", { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ - responses: [response1, response2], - encoding: "base64", - }), - }); - - expect(res.status).toBe(200); - const result = await res.json(); - expect(result.success).toBe(true); - expect(result.totalResponses).toBe(2); - }); - - test("Scenario interceptor returns responses sequentially then passes through", async () => { - const client = createClient({ - socket: { - host: "127.0.0.1", - port: proxy.config.listenPort, - }, - }); - - await client.connect(); - await new Promise((resolve) => setTimeout(resolve, 100)); - - // Set up scenario with 2 responses - const scenarioRes = await app.request("/scenarios", { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ - responses: ["+SCENARIO1\r\n", "+SCENARIO2\r\n"], - encoding: "raw", - }), - }); - - expect(scenarioRes.status).toBe(200); - - // First command should get first scenario response - const result1 = await client.sendCommand(["PING"]); - expect(result1).toBe("SCENARIO1" as unknown as SimpleStringReply); - - // Second command should get second scenario response - const result2 = await client.sendCommand(["PING"]); - expect(result2).toBe("SCENARIO2" as unknown as SimpleStringReply); - - // Third command should pass through to real server - const result3 = await client.sendCommand(["PING"]); - expect(result3).toBe("PONG" as unknown as SimpleStringReply); - - // Fourth command should also pass through - const result4 = await client.sendCommand(["FOO"]); - expect(result4).toBe("BAR" as unknown as SimpleStringReply); - - await client.disconnect(); - }); -}); diff --git a/src/scenarios/helpers.ts b/src/scenarios/helpers.ts new file mode 100644 index 0000000..1a24418 --- /dev/null +++ b/src/scenarios/helpers.ts @@ -0,0 +1,143 @@ +import { + type InterceptorDescription, + type InterceptorState, + type Next, + type ProxyConfig, + RedisProxy, + type SendResult, +} from "redis-monorepo/packages/test-utils/lib/proxy/redis-proxy"; +import type ProxyStore from "../proxy-store"; +import { makeId } from "../proxy-store"; + +/** + * Starts a new proxy with the given configuration + */ +export function startNewProxy(config: ProxyConfig): RedisProxy { + const proxy = new RedisProxy(config); + proxy.start().catch(console.error); + return proxy; +} + +/** + * Adds a new node to the proxy store + */ +export function addNode( + proxyStore: ProxyStore, + config: ProxyConfig, +): { nodeId: string; proxy: RedisProxy } { + const nodeId = makeId(config.targetHost, config.targetPort, config.listenPort); + const proxy = startNewProxy(config); + proxyStore.add(nodeId, proxy); + return { nodeId, proxy }; +} + +/** + * Sends a buffer to all clients across all proxies + */ +export function sendToAllClients(proxyStore: ProxyStore, buffer: Buffer): SendResult[] { + const results: SendResult[] = []; + for (const proxy of proxyStore.proxies) { + results.push(...proxy.sendToAllClients(buffer)); + } + return results; +} + +/** + * Creates a cluster slots interceptor that returns a custom list of proxies + */ +export function createCustomClusterSlotsInterceptor( + proxiesToInclude: RedisProxy[], +): InterceptorDescription { + return { + name: "cluster-simulation-interceptor", + fn: async (data: Buffer, next: Next, state: InterceptorState) => { + state.invokeCount++; + + if (data.toString().toLowerCase() !== "*2\r\n$7\r\ncluster\r\n$5\r\nslots\r\n") { + return next(data); + } + + state.matchCount++; + + const slotLength = Math.floor(16384 / proxiesToInclude.length); + + let current = -1; + const mapping = proxiesToInclude.map((proxy, i) => { + const from = current + 1; + const to = i === proxiesToInclude.length - 1 ? 16383 : current + slotLength; + current = to; + const id = `proxy-id-${proxy.config.listenPort}`; + return `*3\r\n:${from}\r\n:${to}\r\n*3\r\n$${proxy.config.listenHost.length}\r\n${proxy.config.listenHost}\r\n:${proxy.config.listenPort}\r\n$${id.length}\r\n${id}\r\n`; + }); + + const response = `*${proxiesToInclude.length}\r\n${mapping.join("")}`; + return Buffer.from(response); + }, + }; +} + +/** + * Picks a random element from an array + */ +export function pickRandom(array: T[]): T | undefined { + if (array.length === 0) return undefined; + return array[Math.floor(Math.random() * array.length)]; +} + +/** + * Finds the next available port by incrementing from the highest existing port + */ +export function findNextAvailablePort(proxies: RedisProxy[]): number { + const ports = proxies.map((p) => p.config.listenPort); + return Math.max(...ports) + 1; +} + +/** + * Builds an SMIGRATING notification in RESP3 format + * This notifies clients that slots are about to be migrated + */ +export function buildSMigratingNotification(slotRanges: string, seqId: number = 1): Buffer { + const response = `>3\r\n+SMIGRATING\r\n:${seqId}\r\n+${slotRanges}\r\n`; + return Buffer.from(response); +} + +/** + * Builds an SMIGRATED notification in RESP3 format + * This notifies clients that slots have been migrated to different nodes + */ +export function buildSMigratedNotification( + movedSlotsByDestination: Array<{ + targetNode: { host: string; port: number }; + slotRanges: string; // e.g., "0-5460" or "0-100,200-300,500" + }>, + seqId: number = 1, +): Buffer { + if (movedSlotsByDestination.length === 0) { + throw new Error("No slots to migrate"); + } + + const entries = movedSlotsByDestination.map(({ targetNode, slotRanges }) => { + const hostPort = `${targetNode.host}:${targetNode.port}`; + return `*2\r\n+${hostPort}\r\n+${slotRanges}\r\n`; + }); + + const response = `>3\r\n+SMIGRATED\r\n:${seqId}\r\n*${movedSlotsByDestination.length}\r\n${entries.join("")}`; + + return Buffer.from(response); +} + +/** + * Gets the slot ranges assigned to a specific proxy based on cluster slot distribution + */ +export function getSlotRangesForProxy(proxy: RedisProxy, allProxies: RedisProxy[]): string { + const proxyIndex = allProxies.indexOf(proxy); + if (proxyIndex === -1) { + throw new Error("Proxy not found in the list"); + } + + const slotLength = Math.floor(16384 / allProxies.length); + const from = proxyIndex * slotLength; + const to = proxyIndex === allProxies.length - 1 ? 16383 : from + slotLength - 1; + + return `${from}-${to}`; +} diff --git a/src/scenarios/sequence-gen.ts b/src/scenarios/sequence-gen.ts new file mode 100644 index 0000000..8d21a31 --- /dev/null +++ b/src/scenarios/sequence-gen.ts @@ -0,0 +1,4 @@ +let id = 1; +export function getNextSequenceId() { + return id++; +} diff --git a/src/util.ts b/src/util.ts index cabade3..0ca5d74 100644 --- a/src/util.ts +++ b/src/util.ts @@ -24,11 +24,6 @@ export const connectionIdsQuerySchema = z.object({ encoding: z.enum(["base64", "raw"]).default("base64"), }); -export const scenarioSchema = z.object({ - responses: z.array(z.string()).min(1, "At least one response is required"), - encoding: z.enum(["base64", "raw"]).default("base64"), -}); - export const interceptorSchema = z.object({ name: z.string(), encoding: z.enum(["raw", "base64"]), @@ -36,6 +31,86 @@ export const interceptorSchema = z.object({ response: z.string(), }); +export const slotMigrateEffectSchema = z.object({ + effect: z.enum(["add", "remove", "remove-add", "slot-shuffle"]), +}); + +export type SlotMigrateEffect = z.infer["effect"]; + +export interface ActionTriggerRequirement { + dbconfig: Record & { name: string }; + cluster: { min_nodes: number; actual_nodes: number }; + oss_cluster_api: { ip_type: string; endpoint_type: string }; + description: string; +} + +export interface ActionTrigger { + name: string; + description: string; + requirements: ActionTriggerRequirement[]; +} + +export interface ListActionTriggersResponse { + effect: string; + cluster: { index: number; nodes: number }; + triggers: ActionTrigger[]; +} + +// Action types matching re_fault_injector ActionType enum +export const actionTypeSchema = z.enum([ + "dmc_restart", + "failover", + "reshard", + "sequence_of_actions", + "network_failure", + "network_latency", + "execute_rlutil_command", + "execute_rladmin_command", + "enable_entraid", + "upgrade", + "wait", + "wait_for_database_active", + "migrate", + "bind", + "update_cluster_config", + "delete_database", + "create_database", + "shard_failure", + "node_failure", + "node_remove", + "proxy_failure", + "cluster_failure", + "slot_migrate", + "topology_change_standalone", + "reset_cluster", + "collect_debuginfo", +]); + +export type ActionType = z.infer; + +export const actionRequestSchema = z.object({ + type: actionTypeSchema, + parameters: z.record(z.string(), z.unknown()), +}); + +export type ActionRequest = z.infer; + +export const actionIdParamSchema = z.object({ + action_id: z.string(), +}); + +export type ActionStatus = "pending" | "running" | "success" | "failed" | "unknown"; + +export interface ActionRecord { + id: string; + type: ActionType; + parameters: Record; + status: ActionStatus; + submittedAt: Date; + error?: string | null; + output?: unknown; +} + export function parseBuffer(data: string, encoding: "base64" | "raw"): Buffer { switch (encoding) { case "base64":