From f1c7454e4779a8c5bb3c5282086654653e45a9a6 Mon Sep 17 00:00:00 2001 From: Satyam Trivedi Date: Thu, 22 Oct 2026 16:58:09 +0530 Subject: [PATCH 1/2] fix(aws-batch): report multi-node parallel job nodes DescribeJobs now resolves node job IDs ("#") and ListJobs with multiNodeJobId lists the job's nodes instead of an empty list. Each node carries nodeDetails (nodeIndex, isMainNode) and the container, ECS or EKS properties of the node range that covers it; ListJobs summaries carry nodeProperties (isMainNode, numNodes, nodeIndex). Nodes are derived from the job record, the same way array children are, so there is no new stored state. Every node follows the job's lifecycle and, like array children, sends no state-change events of its own. --- docs/coverage/aws/batch.md | 7 +- docs/coverage/nongoals/batch.md | 7 +- providers/aws/batch/job_lifecycle.go | 5 + providers/aws/batch/job_nodes.go | 167 +++++++++++++++++ providers/aws/batch/job_nodes_test.go | 255 ++++++++++++++++++++++++++ providers/aws/batch/jobs.go | 26 +-- server/aws/batch/jobs_sdk_test.go | 90 +++++++++ server/aws/batch/types_jobs.go | 63 +++++-- services/batch/driver/types.go | 14 +- 9 files changed, 602 insertions(+), 32 deletions(-) create mode 100644 providers/aws/batch/job_nodes.go create mode 100644 providers/aws/batch/job_nodes_test.go diff --git a/docs/coverage/aws/batch.md b/docs/coverage/aws/batch.md index 2dbae14ca..f6f8285f1 100644 --- a/docs/coverage/aws/batch.md +++ b/docs/coverage/aws/batch.md @@ -36,8 +36,9 @@ AWS's `batch` service · portable interface `driver.Batch` · [AWS index](./READ - Jobs run no workload: a job always succeeds (container `exitCode` 0) once its dependencies succeed. By default it is `SUCCEEDED` as soon as it is submitted; with `--async-settle` it spends about one second each in `SUBMITTED`, `RUNNABLE` and `STARTING`, then five seconds in `RUNNING` (and stays `PENDING` while a dependency is unfinished). `attempts` is always empty, and retry strategies and timeouts are stored and echoed but never applied. - `CancelJob` on a `PENDING` job takes effect at once. Real Batch waits for the job's dependencies to finish first. -- Array jobs: children are derived from the parent, not stored one by one. `SEQUENTIAL` and `N_TO_N` dependencies are honoured. An array parent reports `PENDING` while any child is unfinished. Multi-node parallel jobs run as a single job: `ListJobs` with `multiNodeJobId` returns an empty list. +- Array jobs: children are derived from the parent, not stored one by one. `SEQUENTIAL` and `N_TO_N` dependencies are honoured. An array parent reports `PENDING` while any child is unfinished. +- Multi-node parallel jobs: nodes are derived from the job, not stored one by one. `DescribeJobs` on `#` and `ListJobs` with `multiNodeJobId` return each node with `nodeDetails` (`nodeIndex`, `isMainNode`) and the container, ECS or EKS properties of its node range. Every node, main or child, has the job's own status, reason and times; real Batch starts the child nodes only once the main node is `RUNNING`, and a child node can finish or fail on its own. `CancelJob` and `TerminateJob` take the job ID, not a node ID. - Fair-share scheduling policies are stored, validated and bound to queues, but they never change the order in which jobs run. Jobs never stay `RUNNABLE` because no capacity is free, so "Batch Job Queue Blocked" events are never sent. -- "Batch Job State Change" events go to the EventBridge default bus on the serve scheduler's tick (and on any Batch job API call), one per status the job passed through; none is sent for the initial submission. In library mode, nothing ticks, so events are published when a job API call sees the change or when `Tick` is called. A job that fails reports only `FAILED`, without the earlier steps. Array children send no events of their own. Real Batch sends no state-change events for compute environments or job queues, so the emulator sends none either. +- "Batch Job State Change" events go to the EventBridge default bus on the serve scheduler's tick (and on any Batch job API call), one per status the job passed through; none is sent for the initial submission. In library mode, nothing ticks, so events are published when a job API call sees the change or when `Tick` is called. A job that fails reports only `FAILED`, without the earlier steps. Array children and multi-node nodes send no events of their own. Real Batch sends no state-change events for compute environments or job queues, so the emulator sends none either. - Error messages are written by the emulator. The exception type (`ClientException`, HTTP 400) is the documented one. -- `containerOverrides`, `ecsPropertiesOverride`, `eksPropertiesOverride` and `nodeOverrides` are merged into the job detail `DescribeJobs` returns (environment variables by name, resource requirements by type, other fields replaced). Overrides are not checked against compute-environment platform rules (for example `nodeOverrides` on Fargate), and multi-node jobs report no per-node `nodeDetails`. +- `containerOverrides`, `ecsPropertiesOverride`, `eksPropertiesOverride` and `nodeOverrides` are merged into the job detail `DescribeJobs` returns (environment variables by name, resource requirements by type, other fields replaced). Overrides are not checked against compute-environment platform rules (for example `nodeOverrides` on Fargate). diff --git a/docs/coverage/nongoals/batch.md b/docs/coverage/nongoals/batch.md index 1cee8c5cd..2dd21a158 100644 --- a/docs/coverage/nongoals/batch.md +++ b/docs/coverage/nongoals/batch.md @@ -1,7 +1,8 @@ - Jobs run no workload: a job always succeeds (container `exitCode` 0) once its dependencies succeed. By default it is `SUCCEEDED` as soon as it is submitted; with `--async-settle` it spends about one second each in `SUBMITTED`, `RUNNABLE` and `STARTING`, then five seconds in `RUNNING` (and stays `PENDING` while a dependency is unfinished). `attempts` is always empty, and retry strategies and timeouts are stored and echoed but never applied. - `CancelJob` on a `PENDING` job takes effect at once. Real Batch waits for the job's dependencies to finish first. -- Array jobs: children are derived from the parent, not stored one by one. `SEQUENTIAL` and `N_TO_N` dependencies are honoured. An array parent reports `PENDING` while any child is unfinished. Multi-node parallel jobs run as a single job: `ListJobs` with `multiNodeJobId` returns an empty list. +- Array jobs: children are derived from the parent, not stored one by one. `SEQUENTIAL` and `N_TO_N` dependencies are honoured. An array parent reports `PENDING` while any child is unfinished. +- Multi-node parallel jobs: nodes are derived from the job, not stored one by one. `DescribeJobs` on `#` and `ListJobs` with `multiNodeJobId` return each node with `nodeDetails` (`nodeIndex`, `isMainNode`) and the container, ECS or EKS properties of its node range. Every node, main or child, has the job's own status, reason and times; real Batch starts the child nodes only once the main node is `RUNNING`, and a child node can finish or fail on its own. `CancelJob` and `TerminateJob` take the job ID, not a node ID. - Fair-share scheduling policies are stored, validated and bound to queues, but they never change the order in which jobs run. Jobs never stay `RUNNABLE` because no capacity is free, so "Batch Job Queue Blocked" events are never sent. -- "Batch Job State Change" events go to the EventBridge default bus on the serve scheduler's tick (and on any Batch job API call), one per status the job passed through; none is sent for the initial submission. In library mode, nothing ticks, so events are published when a job API call sees the change or when `Tick` is called. A job that fails reports only `FAILED`, without the earlier steps. Array children send no events of their own. Real Batch sends no state-change events for compute environments or job queues, so the emulator sends none either. +- "Batch Job State Change" events go to the EventBridge default bus on the serve scheduler's tick (and on any Batch job API call), one per status the job passed through; none is sent for the initial submission. In library mode, nothing ticks, so events are published when a job API call sees the change or when `Tick` is called. A job that fails reports only `FAILED`, without the earlier steps. Array children and multi-node nodes send no events of their own. Real Batch sends no state-change events for compute environments or job queues, so the emulator sends none either. - Error messages are written by the emulator. The exception type (`ClientException`, HTTP 400) is the documented one. -- `containerOverrides`, `ecsPropertiesOverride`, `eksPropertiesOverride` and `nodeOverrides` are merged into the job detail `DescribeJobs` returns (environment variables by name, resource requirements by type, other fields replaced). Overrides are not checked against compute-environment platform rules (for example `nodeOverrides` on Fargate), and multi-node jobs report no per-node `nodeDetails`. +- `containerOverrides`, `ecsPropertiesOverride`, `eksPropertiesOverride` and `nodeOverrides` are merged into the job detail `DescribeJobs` returns (environment variables by name, resource requirements by type, other fields replaced). Overrides are not checked against compute-environment platform rules (for example `nodeOverrides` on Fargate). diff --git a/providers/aws/batch/job_lifecycle.go b/providers/aws/batch/job_lifecycle.go index f2c319853..533caa319 100644 --- a/providers/aws/batch/job_lifecycle.go +++ b/providers/aws/batch/job_lifecycle.go @@ -105,6 +105,11 @@ func cloneJob(j driver.Job) driver.Job { j.ArrayProperties = &ap } + if j.NodeDetails != nil { + nd := *j.NodeDetails + j.NodeDetails = &nd + } + j.Parameters = copyTags(j.Parameters) j.Container = copyRaw(j.Container) j.NodeProperties = copyRaw(j.NodeProperties) diff --git a/providers/aws/batch/job_nodes.go b/providers/aws/batch/job_nodes.go new file mode 100644 index 000000000..70cf97da6 --- /dev/null +++ b/providers/aws/batch/job_nodes.go @@ -0,0 +1,167 @@ +package batch + +import ( + "encoding/json" + "math" + "strconv" + "strings" + + "github.com/stackshy/cloudemu/v2/services/batch/driver" +) + +// nodeSeparator joins a multi-node parallel job's ID and a node index: node i +// of job J is "J#i" (Multi-node parallel jobs, Batch User Guide). +const nodeSeparator = "#" + +// nodeLayout is the node count, main node and node ranges of a multi-node +// parallel job, read from its nodeProperties (overrides already applied). +type nodeLayout struct { + numNodes int32 + mainNode int32 + ranges []jsonObject +} + +// nodeLayoutWire decodes the part of nodeProperties a node layout needs. The +// int32 fields make an out-of-range numNodes or mainNode a decode error. +type nodeLayoutWire struct { + NumNodes int32 `json:"numNodes"` + MainNode int32 `json:"mainNode"` + NodeRangeProperties []jsonObject `json:"nodeRangeProperties"` +} + +// parseNodeLayout reads a job's node layout. ok is false for a job that isn't +// a multi-node parallel job. +func parseNodeLayout(props json.RawMessage) (nodeLayout, bool) { + if !isSet(props) { + return nodeLayout{}, false + } + + var w nodeLayoutWire + if json.Unmarshal(props, &w) != nil || w.NumNodes < 1 || w.MainNode < 0 || w.MainNode >= w.NumNodes { + return nodeLayout{}, false + } + + return nodeLayout{numNodes: w.NumNodes, mainNode: w.MainNode, ranges: w.NodeRangeProperties}, true +} + +// rangeFor returns the node range whose targetNodes covers node i, or nil. +func (l *nodeLayout) rangeFor(i int32) jsonObject { + for _, r := range l.ranges { + var target string + _ = json.Unmarshal(r["targetNodes"], &target) + + nr, ok := parseTargetNodes(target, int(l.numNodes)) + if ok && int(i) >= nr.start && int(i) <= nr.end { + return r + } + } + + return nil +} + +// splitNodeID splits "parent#index" into its parts. +func splitNodeID(id string) (parent string, idx int32, ok bool) { + i := strings.LastIndex(id, nodeSeparator) + if i <= 0 { + return "", 0, false + } + + n, err := strconv.ParseInt(id[i+1:], 10, 32) + if err != nil || n < 0 { + return "", 0, false + } + + return id[:i], int32(n), true +} + +// lookupNode resolves a node job ID "#" to its multi-node +// job's record, the node index and the job's node layout. +func (m *Mock) lookupNode(id string) (rec jobRecord, idx int32, layout nodeLayout, ok bool) { + parent, i, isNode := splitNodeID(id) + if !isNode { + return jobRecord{}, 0, nodeLayout{}, false + } + + r, found := m.jobs.Get(parent) + if !found { + return jobRecord{}, 0, nodeLayout{}, false + } + + l, isMNP := parseNodeLayout(r.Job.NodeProperties) + if !isMNP || i >= l.numNodes { + return jobRecord{}, 0, nodeLayout{}, false + } + + return r, i, l, true +} + +// renderNode builds node i of a multi-node parallel job. Nodes are not stored: +// each node follows the job's own lifecycle (status, reason, times and +// cancel/terminate flags), carries nodeDetails, and reports the container, +// ECS or EKS properties of the node range that covers it instead of the job's +// nodeProperties. +func (o *observer) renderNode(rec *jobRecord, l *nodeLayout, i int32) driver.Job { + j := o.render(rec, -1) + + suffix := nodeSeparator + strconv.Itoa(int(i)) + j.JobID += suffix + j.JobARN += suffix + j.NodeProperties = nil + j.NodeDetails = &driver.NodeDetails{NodeIndex: i, IsMainNode: i == l.mainNode, NumNodes: l.numNodes} + + r := l.rangeFor(i) + j.Container = copyRaw(r["container"]) + j.EcsProperties = copyRaw(r["ecsProperties"]) + j.EksProperties = copyRaw(r["eksProperties"]) + + if j.Status == driver.JobStatusSucceeded { + j.Container = withExitCode(j.Container) + } + + return j +} + +// nodes lists the nodes of a multi-node parallel job by index, resuming after +// the cursor index. Every node shares the job's name, definition, creation +// time, share identifier and status, so a status or filter that rejects one +// node rejects them all: the first node decides and the loop stays bounded by +// the page size. +func (l *jobLister) nodes(m *Mock) *driver.ListJobsOutput { + out := &driver.ListJobsOutput{Jobs: []driver.Job{}} + + rec, ok := m.jobs.Get(l.in.MultiNodeJobID) + if !ok { + return out + } + + layout, isMNP := parseNodeLayout(rec.Job.NodeProperties) + + next := l.after + 1 + if !isMNP || next < 0 || next > math.MaxInt32 || next >= int64(layout.numNodes) { + return out + } + + start := int32(next) + + first := l.o.renderNode(&rec, &layout, start) + if !l.keeps(&first) { + return out + } + + for i := start; i < layout.numNodes; i++ { + if len(out.Jobs) == l.limit { + out.NextToken = encodeListCursor(int64(i - 1)) + + break + } + + out.Jobs = append(out.Jobs, l.o.renderNode(&rec, &layout, i)) + } + + return out +} + +// keeps reports whether a listed job passes the status and filters. +func (l *jobLister) keeps(j *driver.Job) bool { + return (!l.useStatus || j.Status == l.status) && matchesFilters(j, l.in.Filters) +} diff --git a/providers/aws/batch/job_nodes_test.go b/providers/aws/batch/job_nodes_test.go new file mode 100644 index 000000000..6e34a3419 --- /dev/null +++ b/providers/aws/batch/job_nodes_test.go @@ -0,0 +1,255 @@ +package batch_test + +import ( + "context" + "encoding/json" + "strings" + "testing" + "time" + + "github.com/stackshy/cloudemu/v2/config" + "github.com/stackshy/cloudemu/v2/providers/aws/batch" + "github.com/stackshy/cloudemu/v2/services/batch/driver" +) + +// ec2Queue is the EC2-backed job queue setupEC2Queue creates. +const ec2Queue = "q-ec2" + +// setupEC2Queue creates an EC2 compute environment and the job queue ec2Queue +// on it (multi-node parallel jobs need EC2 resources). +func setupEC2Queue(t *testing.T, m *batch.Mock) { + t.Helper() + + ctx := context.Background() + + if _, err := m.CreateComputeEnvironment(ctx, driver.CreateComputeEnvironmentInput{ + Name: "ce-ec2", Type: driver.CETypeManaged, + ComputeResources: json.RawMessage(`{"type":"EC2","minvCpus":0,"maxvCpus":16,` + + `"instanceTypes":["optimal"],"subnets":["subnet-1"],"securityGroupIds":["sg-1"]}`), + }); err != nil { + t.Fatalf("create EC2 compute environment: %v", err) + } + + if _, err := m.CreateJobQueue(ctx, driver.CreateJobQueueInput{ + Name: ec2Queue, Priority: 1, + ComputeEnvironmentOrder: []driver.ComputeEnvironmentOrder{{Order: 1, ComputeEnvironment: "ce-ec2"}}, + }); err != nil { + t.Fatalf("create EC2 queue: %v", err) + } +} + +// registerMNP registers "mnp": three nodes, main node 1, nodes 0-1 on image +// "lead" and node 2 on image "worker". +func registerMNP(t *testing.T, m *batch.Mock) { + t.Helper() + + registerJD(t, m, driver.RegisterJobDefinitionInput{ + Name: "mnp", Type: driver.JDTypeMultiNode, + NodeProperties: json.RawMessage(`{"numNodes":3,"mainNode":1,"nodeRangeProperties":[ + {"targetNodes":"0:1","container":{"image":"lead"}}, + {"targetNodes":"2:","container":{"image":"worker"}}]}`), + }) +} + +func describeAll(t *testing.T, m *batch.Mock, ids ...string) []driver.Job { + t.Helper() + + jobs, err := m.DescribeJobs(context.Background(), ids) + if err != nil { + t.Fatalf("DescribeJobs: %v", err) + } + + return jobs +} + +func containerImage(t *testing.T, j *driver.Job) string { + t.Helper() + + img, _ := decodeMap(t, j.Container)["image"].(string) + + return img +} + +func TestMultiNodeJobDescribesEachNode(t *testing.T) { + m, fc := newAsyncMock(t) + setupEC2Queue(t, m) + registerMNP(t, m) + + j := submit(t, m, driver.SubmitJobInput{JobQueue: ec2Queue, JobDefinition: "mnp"}) + + parent := describe(t, m, j.JobID) + if parent.NodeDetails != nil || !strings.Contains(string(parent.NodeProperties), `"numNodes":3`) { + t.Fatalf("parent should carry nodeProperties and no nodeDetails: %+v", parent) + } + + nodes := describeAll(t, m, j.JobID+"#0", j.JobID+"#1", j.JobID+"#2", j.JobID+"#3", j.JobID+"#-1") + if len(nodes) != 3 { + t.Fatalf("want nodes #0-#2 (and #3, #-1 omitted), got %d", len(nodes)) + } + + wantImage := []string{"lead", "lead", "worker"} + + for i := range nodes { + n := &nodes[i] + if n.NodeDetails == nil || int(n.NodeDetails.NodeIndex) != i || n.NodeDetails.IsMainNode != (i == 1) { + t.Fatalf("node %d nodeDetails = %+v", i, n.NodeDetails) + } + + if n.JobARN != j.JobARN+"#"+string(rune('0'+i)) || n.NodeProperties != nil || n.Status != driver.JobStatusSubmitted { + t.Fatalf("node %d = %+v", i, n) + } + + if got := containerImage(t, n); got != wantImage[i] { + t.Fatalf("node %d container image = %q, want %q", i, got, wantImage[i]) + } + } + + // Nodes follow the job's lifecycle. + fc.Advance(3 * time.Second) + + if n := describe(t, m, j.JobID+"#2"); n.Status != driver.JobStatusRunning || n.StartedAt == 0 { + t.Fatalf("node #2 while the job runs = %s (startedAt %d)", n.Status, n.StartedAt) + } + + fc.Advance(10 * time.Second) + + p := describe(t, m, j.JobID) + main := describe(t, m, j.JobID+"#1") + + if p.Status != driver.JobStatusSucceeded || main.Status != driver.JobStatusSucceeded || + main.StoppedAt != p.StoppedAt || decodeMap(t, main.Container)["exitCode"] != float64(0) { + t.Fatalf("finished job %s / main node %+v", p.Status, main) + } +} + +func TestMultiNodeNodesFollowCancel(t *testing.T) { + m, _ := newAsyncMock(t) + setupEC2Queue(t, m) + registerMNP(t, m) + + j := submit(t, m, driver.SubmitJobInput{JobQueue: ec2Queue, JobDefinition: "mnp"}) + + if err := m.CancelJob(context.Background(), j.JobID, "stop"); err != nil { + t.Fatal(err) + } + + n := describe(t, m, j.JobID+"#0") + if n.Status != driver.JobStatusFailed || !n.IsCancelled || n.StatusReason != "stop" { + t.Fatalf("node of a canceled job = %+v", n) + } +} + +func TestListJobsByMultiNodeJobID(t *testing.T) { + m, _ := newAsyncMock(t) + setupEC2Queue(t, m) + registerMNP(t, m) + registerJD(t, m, driver.RegisterJobDefinitionInput{ + Name: "single", Type: driver.JDTypeContainer, ContainerProperties: json.RawMessage(`{"image":"x"}`), + }) + + j := submit(t, m, driver.SubmitJobInput{JobName: "mnpjob", JobQueue: ec2Queue, JobDefinition: "mnp"}) + single := submit(t, m, driver.SubmitJobInput{JobQueue: ec2Queue, JobDefinition: "single"}) + + all, err := listJobs(m, driver.ListJobsInput{MultiNodeJobID: j.JobID, JobStatus: driver.JobStatusSubmitted}) + if err != nil || len(all) != 3 { + t.Fatalf("ListJobs multiNodeJobId = %d jobs, %v", len(all), err) + } + + for i := range all { + if all[i].JobID != j.JobID+"#"+string(rune('0'+i)) || all[i].NodeDetails == nil || + all[i].NodeDetails.NumNodes != 3 || all[i].NodeDetails.IsMainNode != (i == 1) { + t.Fatalf("node %d = %+v", i, all[i]) + } + } + + // Pages of two. + max2 := int32(2) + + page, err := m.ListJobs(context.Background(), driver.ListJobsInput{ + MultiNodeJobID: j.JobID, JobStatus: driver.JobStatusSubmitted, MaxResults: &max2, + }) + if err != nil || len(page.Jobs) != 2 || page.NextToken == "" { + t.Fatalf("first page = %+v, %v", page, err) + } + + rest, err := m.ListJobs(context.Background(), driver.ListJobsInput{ + MultiNodeJobID: j.JobID, JobStatus: driver.JobStatusSubmitted, MaxResults: &max2, NextToken: page.NextToken, + }) + if err != nil || len(rest.Jobs) != 1 || rest.Jobs[0].JobID != j.JobID+"#2" || rest.NextToken != "" { + t.Fatalf("second page = %+v, %v", rest, err) + } + + empty := map[string]driver.ListJobsInput{ + "default RUNNING status": {MultiNodeJobID: j.JobID}, + "non-matching job name": {MultiNodeJobID: j.JobID, Filters: []driver.JobFilter{{Name: driver.FilterJobName, Values: []string{"nomatch"}}}}, + "unknown job": {MultiNodeJobID: "nope", JobStatus: driver.JobStatusSubmitted}, + "job that isn't multinode": {MultiNodeJobID: single.JobID, JobStatus: driver.JobStatusSubmitted}, + } + + for name, in := range empty { + got, err := listJobs(m, in) + if err != nil || len(got) != 0 { + t.Errorf("%s: got %d jobs, %v", name, len(got), err) + } + } + + if got, _ := listJobs(m, driver.ListJobsInput{ + MultiNodeJobID: j.JobID, Filters: []driver.JobFilter{{Name: driver.FilterJobName, Values: []string{"mnp*"}}}, + }); len(got) != 3 { + t.Fatalf("matching filter returned %d nodes", len(got)) + } +} + +func TestMultiNodeNodesSendNoEvents(t *testing.T) { + m := newMock(t) + pub := &recordingPublisher{} + m.SetEventPublisher(pub) + setupEC2Queue(t, m) + registerMNP(t, m) + + j := submit(t, m, driver.SubmitJobInput{JobQueue: ec2Queue, JobDefinition: "mnp"}) + describeAll(t, m, j.JobID+"#0", j.JobID+"#1") + + pub.mu.Lock() + defer pub.mu.Unlock() + + if len(pub.events) != 4 { + t.Fatalf("want the job's 4 events only, got %d", len(pub.events)) + } + + for _, e := range pub.events { + if len(e.resources) != 1 || e.resources[0] != j.JobARN { + t.Fatalf("event for something other than the job: %v", e.resources) + } + } +} + +func TestMultiNodeNodesSurviveSnapshot(t *testing.T) { + src, fc := newAsyncMock(t) + setupEC2Queue(t, src) + registerMNP(t, src) + ctx := context.Background() + + j := submit(t, src, driver.SubmitJobInput{ + JobQueue: ec2Queue, JobDefinition: "mnp", + NodeOverrides: json.RawMessage(`{"nodePropertyOverrides":[{"targetNodes":"2:","containerOverrides":{"command":["go"]}}]}`), + }) + + data, err := src.Snapshot(ctx, false) + if err != nil { + t.Fatal(err) + } + + dst := batch.New(config.NewOptions( + config.WithRegion("us-east-1"), config.WithAccountID("000000000000"), + config.WithClock(fc), config.WithAsyncSettle(), + )) + if err := dst.Restore(ctx, data); err != nil { + t.Fatal(err) + } + + n := describe(t, dst, j.JobID+"#2") + if n.NodeDetails == nil || n.NodeDetails.NodeIndex != 2 || !strings.Contains(string(n.Container), `"command":["go"]`) { + t.Fatalf("restored node = %+v", n) + } +} diff --git a/providers/aws/batch/jobs.go b/providers/aws/batch/jobs.go index 1ad6ca11f..57b599d13 100644 --- a/providers/aws/batch/jobs.go +++ b/providers/aws/batch/jobs.go @@ -474,8 +474,9 @@ func mergeKeyed(base, over json.RawMessage, key string) json.RawMessage { return out } -// DescribeJobs returns the requested jobs (top-level or array-child IDs) in -// request order, each observed now. Unknown IDs are omitted. +// DescribeJobs returns the requested jobs (top-level, array-child ":" or +// multi-node "#" IDs) in request order, each observed now. Unknown IDs +// are omitted. func (m *Mock) DescribeJobs(ctx context.Context, jobIDs []string) ([]driver.Job, error) { if len(jobIDs) == 0 { return nil, cerrors.New(cerrors.InvalidArgument, "jobs is required") @@ -491,12 +492,15 @@ func (m *Mock) DescribeJobs(ctx context.Context, jobIDs []string) ([]driver.Job, out := make([]driver.Job, 0, len(jobIDs)) for _, id := range jobIDs { - rec, idx, ok := m.lookup(id) - if !ok { + if rec, idx, ok := m.lookup(id); ok { + out = append(out, o.render(&rec, idx)) + continue } - out = append(out, o.render(&rec, idx)) + if rec, idx, layout, ok := m.lookupNode(id); ok { + out = append(out, o.renderNode(&rec, &layout, idx)) + } } return out, nil @@ -509,14 +513,15 @@ const ( listJobsFilterPage = 100 ) -// ListJobs lists the jobs of a queue (top-level jobs and array parents) or the -// children of an array job. Queue listings are newest first, as the ListJobs -// reference describes; children are listed by index. Exactly one of jobQueue, +// ListJobs lists the jobs of a queue (top-level jobs and array parents), the +// children of an array job or the nodes of a multi-node parallel job. Queue +// listings are newest first, as the ListJobs reference describes; children and +// nodes are listed by index. Exactly one of jobQueue, // arrayJobId and multiNodeJobId must be given. Without a filter only jobs in // jobStatus (default RUNNING) are returned. // // The page token is a cursor on the listing order (submission sequence, or -// child index), not an offset, so jobs submitted or changing status between +// child or node index), not an offset, so jobs submitted or changing status between // pages never make a page skip or repeat an entry. // //nolint:gocritic // in is the driver ListJobsInput, taken by value to match the interface @@ -544,8 +549,7 @@ func (m *Mock) ListJobs(ctx context.Context, in driver.ListJobsInput) (*driver.L switch { case in.MultiNodeJobID != "": - // Multi-node parallel jobs are not modeled node by node. - return &driver.ListJobsOutput{Jobs: []driver.Job{}}, nil + return l.nodes(m), nil case in.ArrayJobID != "": return l.children(m), nil default: diff --git a/server/aws/batch/jobs_sdk_test.go b/server/aws/batch/jobs_sdk_test.go index 10f29f568..54f2e136f 100644 --- a/server/aws/batch/jobs_sdk_test.go +++ b/server/aws/batch/jobs_sdk_test.go @@ -289,3 +289,93 @@ func TestSDKSubmitJobEcsPropertiesOverride(t *testing.T) { }) requireClientException(t, err, "ecs override on containerProperties definition") } + +// setupSDKMultiNode creates an EC2 queue "q-ec2" and a multinode job definition +// "mnp" (two nodes, main node 0) next to setupSDKQueue's Fargate queue "q". +func setupSDKMultiNode(t *testing.T, c *awsbatch.Client) { + t.Helper() + + ctx := context.Background() + + if _, err := c.CreateComputeEnvironment(ctx, &awsbatch.CreateComputeEnvironmentInput{ + ComputeEnvironmentName: aws.String("ce-ec2"), + Type: batchtypes.CETypeManaged, + ComputeResources: &batchtypes.ComputeResource{ + Type: batchtypes.CRTypeEc2, MaxvCpus: aws.Int32(16), InstanceTypes: []string{"optimal"}, + Subnets: []string{"subnet-1111"}, SecurityGroupIds: []string{"sg-abcd"}, + }, + }); err != nil { + t.Fatalf("CreateComputeEnvironment EC2: %v", err) + } + + if _, err := c.CreateJobQueue(ctx, &awsbatch.CreateJobQueueInput{ + JobQueueName: aws.String("q-ec2"), Priority: aws.Int32(1), + ComputeEnvironmentOrder: []batchtypes.ComputeEnvironmentOrder{ + {Order: aws.Int32(1), ComputeEnvironment: aws.String("ce-ec2")}, + }, + }); err != nil { + t.Fatalf("CreateJobQueue EC2: %v", err) + } + + if _, err := c.RegisterJobDefinition(ctx, &awsbatch.RegisterJobDefinitionInput{ + JobDefinitionName: aws.String("mnp"), + Type: batchtypes.JobDefinitionTypeMultinode, + NodeProperties: &batchtypes.NodeProperties{ + NumNodes: aws.Int32(2), MainNode: aws.Int32(0), + NodeRangeProperties: []batchtypes.NodeRangeProperty{ + {TargetNodes: aws.String("0:"), Container: &batchtypes.ContainerProperties{Image: aws.String("mpi")}}, + }, + }, + }); err != nil { + t.Fatalf("RegisterJobDefinition multinode: %v", err) + } +} + +func TestSDKMultiNodeJobNodes(t *testing.T) { + ctx := context.Background() + c := newBatchClient(t) + setupSDKQueue(t, c, nil) + setupSDKMultiNode(t, c) + + sub, err := c.SubmitJob(ctx, &awsbatch.SubmitJobInput{ + JobName: aws.String("mpi"), JobQueue: aws.String("q-ec2"), JobDefinition: aws.String("mnp"), + NodeOverrides: &batchtypes.NodeOverrides{NumNodes: aws.Int32(3)}, + }) + if err != nil { + t.Fatalf("SubmitJob multinode: %v", err) + } + + id := aws.ToString(sub.JobId) + + desc, err := c.DescribeJobs(ctx, &awsbatch.DescribeJobsInput{Jobs: []string{id, id + "#0", id + "#2", id + "#3"}}) + if err != nil || len(desc.Jobs) != 3 { + t.Fatalf("DescribeJobs: %+v, %v", desc, err) + } + + parent, main, last := desc.Jobs[0], desc.Jobs[1], desc.Jobs[2] + if parent.NodeDetails != nil || parent.NodeProperties == nil || aws.ToInt32(parent.NodeProperties.NumNodes) != 3 { + t.Fatalf("parent: nodeDetails %+v nodeProperties %+v", parent.NodeDetails, parent.NodeProperties) + } + + if main.NodeDetails == nil || !aws.ToBool(main.NodeDetails.IsMainNode) || aws.ToInt32(main.NodeDetails.NodeIndex) != 0 || + aws.ToString(main.JobId) != id+"#0" || main.Container == nil || aws.ToString(main.Container.Image) != "mpi" || + main.Status != parent.Status { + t.Fatalf("main node: %+v", main) + } + + if last.NodeDetails == nil || aws.ToBool(last.NodeDetails.IsMainNode) || aws.ToInt32(last.NodeDetails.NodeIndex) != 2 { + t.Fatalf("node #2: %+v", last.NodeDetails) + } + + list, err := c.ListJobs(ctx, &awsbatch.ListJobsInput{MultiNodeJobId: aws.String(id), JobStatus: batchtypes.JobStatusSucceeded}) + if err != nil || len(list.JobSummaryList) != 3 { + t.Fatalf("ListJobs multiNodeJobId: %+v, %v", list, err) + } + + for i, s := range list.JobSummaryList { + np := s.NodeProperties + if np == nil || aws.ToInt32(np.NodeIndex) != int32(i) || aws.ToInt32(np.NumNodes) != 3 || aws.ToBool(np.IsMainNode) != (i == 0) { + t.Fatalf("summary %d nodeProperties: %+v", i, np) + } + } +} diff --git a/server/aws/batch/types_jobs.go b/server/aws/batch/types_jobs.go index 0e5ce93be..c9b952315 100644 --- a/server/aws/batch/types_jobs.go +++ b/server/aws/batch/types_jobs.go @@ -125,6 +125,21 @@ type arrayPropertiesWire struct { StatusSummary map[string]int32 `json:"statusSummary,omitempty"` } +// nodeDetailsWire is JobDetail.nodeDetails, set on a node of a multi-node +// parallel job. +type nodeDetailsWire struct { + NodeIndex int32 `json:"nodeIndex"` + IsMainNode bool `json:"isMainNode"` +} + +// nodePropertiesSummaryWire is JobSummary.nodeProperties for a node of a +// multi-node parallel job. +type nodePropertiesSummaryWire struct { + IsMainNode bool `json:"isMainNode"` + NumNodes int32 `json:"numNodes"` + NodeIndex int32 `json:"nodeIndex"` +} + type jobDetail struct { JobArn string `json:"jobArn"` JobName string `json:"jobName"` @@ -140,6 +155,7 @@ type jobDetail struct { JobDefinition string `json:"jobDefinition"` Parameters map[string]string `json:"parameters"` Container json.RawMessage `json:"container,omitempty"` + NodeDetails *nodeDetailsWire `json:"nodeDetails,omitempty"` NodeProperties json.RawMessage `json:"nodeProperties,omitempty"` EcsProperties json.RawMessage `json:"ecsProperties,omitempty"` EksProperties json.RawMessage `json:"eksProperties,omitempty"` @@ -179,20 +195,21 @@ type containerSummary struct { } type jobSummary struct { - JobArn string `json:"jobArn"` - JobID string `json:"jobId"` - JobName string `json:"jobName"` - CreatedAt int64 `json:"createdAt,omitempty"` - Status string `json:"status"` - StatusReason string `json:"statusReason,omitempty"` - StartedAt int64 `json:"startedAt,omitempty"` - StoppedAt int64 `json:"stoppedAt,omitempty"` - Container *containerSummary `json:"container,omitempty"` - ArrayProperties *arrayPropertiesWire `json:"arrayProperties,omitempty"` - JobDefinition string `json:"jobDefinition,omitempty"` - ShareIdentifier string `json:"shareIdentifier,omitempty"` - IsCancelled *bool `json:"isCancelled,omitempty"` - IsTerminated *bool `json:"isTerminated,omitempty"` + JobArn string `json:"jobArn"` + JobID string `json:"jobId"` + JobName string `json:"jobName"` + CreatedAt int64 `json:"createdAt,omitempty"` + Status string `json:"status"` + StatusReason string `json:"statusReason,omitempty"` + StartedAt int64 `json:"startedAt,omitempty"` + StoppedAt int64 `json:"stoppedAt,omitempty"` + Container *containerSummary `json:"container,omitempty"` + ArrayProperties *arrayPropertiesWire `json:"arrayProperties,omitempty"` + NodeProperties *nodePropertiesSummaryWire `json:"nodeProperties,omitempty"` + JobDefinition string `json:"jobDefinition,omitempty"` + ShareIdentifier string `json:"shareIdentifier,omitempty"` + IsCancelled *bool `json:"isCancelled,omitempty"` + IsTerminated *bool `json:"isTerminated,omitempty"` } type listJobsResponse struct { @@ -248,6 +265,22 @@ func arrayToWire(ap *driver.ArrayProperties) *arrayPropertiesWire { return &arrayPropertiesWire{Size: ap.Size, Index: ap.Index, StatusSummary: ap.StatusSummary} } +func nodeDetailsToWire(nd *driver.NodeDetails) *nodeDetailsWire { + if nd == nil { + return nil + } + + return &nodeDetailsWire{NodeIndex: nd.NodeIndex, IsMainNode: nd.IsMainNode} +} + +func nodeSummaryToWire(nd *driver.NodeDetails) *nodePropertiesSummaryWire { + if nd == nil { + return nil + } + + return &nodePropertiesSummaryWire{IsMainNode: nd.IsMainNode, NumNodes: nd.NumNodes, NodeIndex: nd.NodeIndex} +} + func boolPtrIf(b bool) *bool { if !b { return nil @@ -282,6 +315,7 @@ func toJobDetail(j *driver.Job) jobDetail { JobDefinition: j.JobDefinition, Parameters: params, Container: j.Container, + NodeDetails: nodeDetailsToWire(j.NodeDetails), NodeProperties: j.NodeProperties, EcsProperties: j.EcsProperties, EksProperties: j.EksProperties, @@ -311,6 +345,7 @@ func toJobSummary(j *driver.Job) jobSummary { StartedAt: j.StartedAt, StoppedAt: j.StoppedAt, ArrayProperties: arrayToWire(j.ArrayProperties), + NodeProperties: nodeSummaryToWire(j.NodeDetails), JobDefinition: j.JobDefinition, ShareIdentifier: j.ShareIdentifier, IsCancelled: boolPtrIf(j.IsCancelled), diff --git a/services/batch/driver/types.go b/services/batch/driver/types.go index ea5d6000b..ad416080a 100644 --- a/services/batch/driver/types.go +++ b/services/batch/driver/types.go @@ -244,6 +244,16 @@ type ArrayProperties struct { StatusSummary map[string]int32 } +// NodeDetails identifies one node of a multi-node parallel job: the node job +// "#" that DescribeJobs and ListJobs (multiNodeJobId) return. +// NumNodes is the job's node count, which ListJobs reports in the node's +// JobSummary nodeProperties; DescribeJobs reports NodeIndex and IsMainNode. +type NodeDetails struct { + NodeIndex int32 + IsMainNode bool + NumNodes int32 +} + // SubmitJobInput describes a job to submit. JobQueue and JobDefinition accept // a name or ARN (a job definition with or without a revision). A nil ArraySize // submits a single (non-array) job. @@ -270,7 +280,8 @@ type SubmitJobInput struct { // (JobDetail shape) at the instant it is observed. Times are epoch // milliseconds; zero means unset. Container, NodeProperties, EcsProperties and // EksProperties are the job definition's blobs (with container overrides -// applied) carried as raw JSON. +// applied) carried as raw JSON. NodeDetails is set only on a node of a +// multi-node parallel job. type Job struct { JobID string JobARN string @@ -284,6 +295,7 @@ type Job struct { StoppedAt int64 DependsOn []JobDependency ArrayProperties *ArrayProperties + NodeDetails *NodeDetails Parameters map[string]string Container json.RawMessage NodeProperties json.RawMessage From fff15afdab883246a6182144f9bbbe529784ce00 Mon Sep 17 00:00:00 2001 From: Satyam Trivedi Date: Thu, 22 Oct 2026 16:58:51 +0530 Subject: [PATCH 2/2] fix(aws-batch): reject multi-node work on Fargate job queues SubmitJob now returns a ClientException when nodeOverrides is sent to a job queue whose compute environments are FARGATE or FARGATE_SPOT, or when a multi-node job definition is submitted to one. The SubmitJob reference says nodeOverrides "isn't applicable to jobs that are running on Fargate resources; use containerOverrides instead", and the RegisterJobDefinition reference says multinode isn't supported for jobs run on Fargate. RegisterJobDefinition also rejects nodeProperties together with the FARGATE platform capability, per the same reference. The override tests that submit multi-node jobs now use an EC2 queue, so they keep testing the override rules rather than the platform rule. --- docs/coverage/aws/batch.md | 2 +- docs/coverage/nongoals/batch.md | 2 +- providers/aws/batch/job_definition.go | 4 ++ providers/aws/batch/job_nodes.go | 79 ++++++++++++++++++++++ providers/aws/batch/job_nodes_test.go | 71 +++++++++++++++++++ providers/aws/batch/jobs.go | 4 ++ providers/aws/batch/jobs_overrides_test.go | 8 ++- server/aws/batch/jobs_sdk_test.go | 30 ++++++++ 8 files changed, 196 insertions(+), 4 deletions(-) diff --git a/docs/coverage/aws/batch.md b/docs/coverage/aws/batch.md index f6f8285f1..f2e828997 100644 --- a/docs/coverage/aws/batch.md +++ b/docs/coverage/aws/batch.md @@ -41,4 +41,4 @@ AWS's `batch` service · portable interface `driver.Batch` · [AWS index](./READ - Fair-share scheduling policies are stored, validated and bound to queues, but they never change the order in which jobs run. Jobs never stay `RUNNABLE` because no capacity is free, so "Batch Job Queue Blocked" events are never sent. - "Batch Job State Change" events go to the EventBridge default bus on the serve scheduler's tick (and on any Batch job API call), one per status the job passed through; none is sent for the initial submission. In library mode, nothing ticks, so events are published when a job API call sees the change or when `Tick` is called. A job that fails reports only `FAILED`, without the earlier steps. Array children and multi-node nodes send no events of their own. Real Batch sends no state-change events for compute environments or job queues, so the emulator sends none either. - Error messages are written by the emulator. The exception type (`ClientException`, HTTP 400) is the documented one. -- `containerOverrides`, `ecsPropertiesOverride`, `eksPropertiesOverride` and `nodeOverrides` are merged into the job detail `DescribeJobs` returns (environment variables by name, resource requirements by type, other fields replaced). Overrides are not checked against compute-environment platform rules (for example `nodeOverrides` on Fargate). +- `containerOverrides`, `ecsPropertiesOverride`, `eksPropertiesOverride` and `nodeOverrides` are merged into the job detail `DescribeJobs` returns (environment variables by name, resource requirements by type, other fields replaced). On a job queue whose compute environments are `FARGATE` or `FARGATE_SPOT`, `SubmitJob` rejects `nodeOverrides` and multi-node job definitions with a `ClientException`, and `RegisterJobDefinition` rejects `nodeProperties` with the `FARGATE` platform capability. Other platform rules are not checked: multi-node jobs on Spot or `UNMANAGED` compute environments, the `instanceType`, `memory` and `vcpus` container overrides on Fargate, and ECS versus EKS properties against the compute environment, since EKS compute environments are not modeled. diff --git a/docs/coverage/nongoals/batch.md b/docs/coverage/nongoals/batch.md index 2dd21a158..48c21cf06 100644 --- a/docs/coverage/nongoals/batch.md +++ b/docs/coverage/nongoals/batch.md @@ -5,4 +5,4 @@ - Fair-share scheduling policies are stored, validated and bound to queues, but they never change the order in which jobs run. Jobs never stay `RUNNABLE` because no capacity is free, so "Batch Job Queue Blocked" events are never sent. - "Batch Job State Change" events go to the EventBridge default bus on the serve scheduler's tick (and on any Batch job API call), one per status the job passed through; none is sent for the initial submission. In library mode, nothing ticks, so events are published when a job API call sees the change or when `Tick` is called. A job that fails reports only `FAILED`, without the earlier steps. Array children and multi-node nodes send no events of their own. Real Batch sends no state-change events for compute environments or job queues, so the emulator sends none either. - Error messages are written by the emulator. The exception type (`ClientException`, HTTP 400) is the documented one. -- `containerOverrides`, `ecsPropertiesOverride`, `eksPropertiesOverride` and `nodeOverrides` are merged into the job detail `DescribeJobs` returns (environment variables by name, resource requirements by type, other fields replaced). Overrides are not checked against compute-environment platform rules (for example `nodeOverrides` on Fargate). +- `containerOverrides`, `ecsPropertiesOverride`, `eksPropertiesOverride` and `nodeOverrides` are merged into the job detail `DescribeJobs` returns (environment variables by name, resource requirements by type, other fields replaced). On a job queue whose compute environments are `FARGATE` or `FARGATE_SPOT`, `SubmitJob` rejects `nodeOverrides` and multi-node job definitions with a `ClientException`, and `RegisterJobDefinition` rejects `nodeProperties` with the `FARGATE` platform capability. Other platform rules are not checked: multi-node jobs on Spot or `UNMANAGED` compute environments, the `instanceType`, `memory` and `vcpus` container overrides on Fargate, and ECS versus EKS properties against the compute environment, since EKS compute environments are not modeled. diff --git a/providers/aws/batch/job_definition.go b/providers/aws/batch/job_definition.go index 1c8bc54d7..ba98dd705 100644 --- a/providers/aws/batch/job_definition.go +++ b/providers/aws/batch/job_definition.go @@ -49,6 +49,10 @@ func (m *Mock) RegisterJobDefinition( return nil, err } + if err := validateJobDefinitionPlatform(&in); err != nil { + return nil, err + } + m.registerMu.Lock() cur, _ := m.jobDefMaxRev.Get(in.Name) diff --git a/providers/aws/batch/job_nodes.go b/providers/aws/batch/job_nodes.go index 70cf97da6..c48dfc25f 100644 --- a/providers/aws/batch/job_nodes.go +++ b/providers/aws/batch/job_nodes.go @@ -6,6 +6,7 @@ import ( "strconv" "strings" + cerrors "github.com/stackshy/cloudemu/v2/errors" "github.com/stackshy/cloudemu/v2/services/batch/driver" ) @@ -13,6 +14,14 @@ import ( // of job J is "J#i" (Multi-node parallel jobs, Batch User Guide). const nodeSeparator = "#" +// Compute resource types that run on Fargate (ComputeResource.type), and the +// job definition platform capability for Fargate. +const ( + crTypeFargate = "FARGATE" + crTypeFargateSpot = "FARGATE_SPOT" + platformFargate = "FARGATE" +) + // nodeLayout is the node count, main node and node ranges of a multi-node // parallel job, read from its nodeProperties (overrides already applied). type nodeLayout struct { @@ -165,3 +174,73 @@ func (l *jobLister) nodes(m *Mock) *driver.ListJobsOutput { func (l *jobLister) keeps(j *driver.Job) bool { return (!l.useStatus || j.Status == l.status) && matchesFilters(j, l.in.Filters) } + +// fargateQueue reports whether a job queue runs its jobs on Fargate: its +// compute environments have FARGATE or FARGATE_SPOT compute resources. A queue +// can't mix Fargate and EC2 compute environments (Job queues on Fargate, Batch +// User Guide), so one Fargate compute environment decides. +func (m *Mock) fargateQueue(q *driver.JobQueue) bool { + for _, o := range q.ComputeEnvironmentOrder { + ce, ok := m.computeEnvs.Get(nameFromARN(kindComputeEnvironment, o.ComputeEnvironment)) + if !ok || !isSet(ce.ComputeResources) { + continue + } + + var cr struct { + Type string `json:"type"` + } + + if json.Unmarshal(ce.ComputeResources, &cr) == nil && (cr.Type == crTypeFargate || cr.Type == crTypeFargateSpot) { + return true + } + } + + return false +} + +// validateJobPlatform rejects a submission the queue's platform can't run. On +// a Fargate queue, nodeOverrides isn't applicable (SubmitJob API reference: +// "use containerOverrides instead") and multi-node parallel jobs aren't +// supported (RegisterJobDefinition API reference, type and nodeProperties). +func (m *Mock) validateJobPlatform(q *driver.JobQueue, jd *driver.JobDefinition, in *driver.SubmitJobInput) error { + if !isSet(in.NodeOverrides) && !isMultiNode(jd) { + return nil + } + + if !m.fargateQueue(q) { + return nil + } + + if isSet(in.NodeOverrides) { + return cerrors.Newf(cerrors.InvalidArgument, + "nodeOverrides isn't applicable to jobs that run on Fargate resources (job queue %q uses Fargate "+ + "compute environments); use containerOverrides instead", q.Name) + } + + return cerrors.Newf(cerrors.InvalidArgument, + "multi-node parallel job definition %q can't run on Fargate resources (job queue %q uses Fargate "+ + "compute environments)", jd.ARN, q.Name) +} + +func isMultiNode(jd *driver.JobDefinition) bool { + return jd.Type == driver.JDTypeMultiNode || isSet(jd.NodeProperties) +} + +// validateJobDefinitionPlatform enforces the RegisterJobDefinition rule that a +// job run on Fargate resources can't be multi-node: nodeProperties (and so +// type multinode) can't be combined with the FARGATE platform capability. +func validateJobDefinitionPlatform(in *driver.RegisterJobDefinitionInput) error { + if !isSet(in.NodeProperties) && in.Type != driver.JDTypeMultiNode { + return nil + } + + for _, c := range in.PlatformCapabilities { + if c == platformFargate { + return cerrors.New(cerrors.InvalidArgument, + "multinode job definitions and nodeProperties aren't supported for jobs that run on Fargate "+ + "resources (platformCapabilities FARGATE)") + } + } + + return nil +} diff --git a/providers/aws/batch/job_nodes_test.go b/providers/aws/batch/job_nodes_test.go index 6e34a3419..dc1ee3882 100644 --- a/providers/aws/batch/job_nodes_test.go +++ b/providers/aws/batch/job_nodes_test.go @@ -8,6 +8,7 @@ import ( "time" "github.com/stackshy/cloudemu/v2/config" + "github.com/stackshy/cloudemu/v2/errors" "github.com/stackshy/cloudemu/v2/providers/aws/batch" "github.com/stackshy/cloudemu/v2/services/batch/driver" ) @@ -253,3 +254,73 @@ func TestMultiNodeNodesSurviveSnapshot(t *testing.T) { t.Fatalf("restored node = %+v", n) } } + +func TestSubmitJobRejectsMultiNodeOnFargate(t *testing.T) { + m := newMock(t) + setupQueue(t, m) // queue "q" is on a FARGATE compute environment + setupEC2Queue(t, m) + registerMNP(t, m) + ctx := context.Background() + + if _, err := m.CreateComputeEnvironment(ctx, driver.CreateComputeEnvironmentInput{ + Name: "ce-spot", Type: driver.CETypeManaged, + ComputeResources: json.RawMessage(`{"type":"FARGATE_SPOT","maxvCpus":4,"subnets":["s"]}`), + }); err != nil { + t.Fatal(err) + } + + if _, err := m.CreateJobQueue(ctx, driver.CreateJobQueueInput{ + Name: "q-spot", Priority: 1, + ComputeEnvironmentOrder: []driver.ComputeEnvironmentOrder{{Order: 1, ComputeEnvironment: "ce-spot"}}, + }); err != nil { + t.Fatal(err) + } + + nodeOverrides := json.RawMessage(`{"nodePropertyOverrides":[{"targetNodes":"0:","containerOverrides":{"command":["x"]}}]}`) + + cases := []struct { + name, queue, jd, want string + overrides json.RawMessage + }{ + {"nodeOverrides on a Fargate queue", "q", "mnp", "nodeOverrides isn't applicable", nodeOverrides}, + {"nodeOverrides with a container definition on Fargate", "q", "jd", "nodeOverrides isn't applicable", nodeOverrides}, + {"multinode definition on a Fargate queue", "q", "mnp", "can't run on Fargate", nil}, + {"multinode definition on a FARGATE_SPOT queue (by ARN)", + "arn:aws:batch:us-east-1:000000000000:job-queue/q-spot", "mnp", "can't run on Fargate", nil}, + } + + for _, tc := range cases { + _, err := m.SubmitJob(ctx, driver.SubmitJobInput{ + JobName: "j", JobQueue: tc.queue, JobDefinition: tc.jd, NodeOverrides: tc.overrides, + }) + if !errors.IsInvalidArgument(err) || !strings.Contains(err.Error(), tc.want) { + t.Errorf("%s: want InvalidArgument %q, got %v", tc.name, tc.want, err) + } + } + + // The same submissions are accepted on an EC2 queue. + submit(t, m, driver.SubmitJobInput{JobQueue: ec2Queue, JobDefinition: "mnp", NodeOverrides: nodeOverrides}) + + // A container job on a Fargate queue is unaffected. + submit(t, m, driver.SubmitJobInput{JobQueue: "q", JobDefinition: "jd"}) +} + +func TestRegisterMultiNodeRejectsFargatePlatform(t *testing.T) { + m := newMock(t) + ctx := context.Background() + props := json.RawMessage(`{"numNodes":2,"mainNode":0,"nodeRangeProperties":[{"targetNodes":"0:","container":{"image":"x"}}]}`) + + for _, typ := range []string{driver.JDTypeMultiNode, driver.JDTypeContainer} { + _, err := m.RegisterJobDefinition(ctx, driver.RegisterJobDefinitionInput{ + Name: "fg", Type: typ, NodeProperties: props, PlatformCapabilities: []string{"FARGATE"}, + ContainerProperties: json.RawMessage(`{"image":"x"}`), + }) + if !errors.IsInvalidArgument(err) { + t.Errorf("%s nodeProperties with FARGATE: want InvalidArgument, got %v", typ, err) + } + } + + registerJD(t, m, driver.RegisterJobDefinitionInput{ + Name: "ec2", Type: driver.JDTypeMultiNode, NodeProperties: props, PlatformCapabilities: []string{"EC2"}, + }) +} diff --git a/providers/aws/batch/jobs.go b/providers/aws/batch/jobs.go index 57b599d13..ca40b4b71 100644 --- a/providers/aws/batch/jobs.go +++ b/providers/aws/batch/jobs.go @@ -58,6 +58,10 @@ func (m *Mock) SubmitJob(ctx context.Context, in driver.SubmitJobInput) (*driver return nil, err } + if platformErr := m.validateJobPlatform(&q, &jd, &in); platformErr != nil { + return nil, platformErr + } + job, err := m.buildJob(&in, &q, &jd) if err != nil { return nil, err diff --git a/providers/aws/batch/jobs_overrides_test.go b/providers/aws/batch/jobs_overrides_test.go index af664d4ba..60e46c2da 100644 --- a/providers/aws/batch/jobs_overrides_test.go +++ b/providers/aws/batch/jobs_overrides_test.go @@ -116,6 +116,7 @@ func TestSubmitJobAppliesEksPropertiesOverride(t *testing.T) { func TestSubmitJobAppliesNodeOverrides(t *testing.T) { m := newMock(t) setupQueue(t, m) + setupEC2Queue(t, m) registerJD(t, m, driver.RegisterJobDefinitionInput{ Name: "mnp", Type: driver.JDTypeMultiNode, @@ -125,7 +126,7 @@ func TestSubmitJobAppliesNodeOverrides(t *testing.T) { }) j := submit(t, m, driver.SubmitJobInput{ - JobDefinition: "mnp", + JobDefinition: "mnp", JobQueue: ec2Queue, NodeOverrides: json.RawMessage(`{"numNodes":4,"nodePropertyOverrides":[ {"targetNodes":"1:","containerOverrides":{"command":["work","--hard"]},"instanceTypes":["c5.large"]}]}`), }) @@ -155,6 +156,7 @@ func TestSubmitJobAppliesNodeOverrides(t *testing.T) { func TestSubmitJobRejectsUnusableOverrides(t *testing.T) { m := newMock(t) setupQueue(t, m) + setupEC2Queue(t, m) ctx := context.Background() registerJD(t, m, driver.RegisterJobDefinitionInput{ @@ -184,7 +186,9 @@ func TestSubmitJobRejectsUnusableOverrides(t *testing.T) { } for name, in := range cases { - in.JobName, in.JobQueue = "j", "q" + // An EC2 queue, so the override rules (not the Fargate platform + // rule) are what rejects each case. + in.JobName, in.JobQueue = "j", ec2Queue if _, err := m.SubmitJob(ctx, in); !errors.IsInvalidArgument(err) { t.Errorf("%s: want InvalidArgument, got %v", name, err) } diff --git a/server/aws/batch/jobs_sdk_test.go b/server/aws/batch/jobs_sdk_test.go index 54f2e136f..eda1b2209 100644 --- a/server/aws/batch/jobs_sdk_test.go +++ b/server/aws/batch/jobs_sdk_test.go @@ -379,3 +379,33 @@ func TestSDKMultiNodeJobNodes(t *testing.T) { } } } + +func TestSDKSubmitJobRejectsNodeOverridesOnFargate(t *testing.T) { + ctx := context.Background() + c := newBatchClient(t) + setupSDKQueue(t, c, nil) + setupSDKMultiNode(t, c) + + _, err := c.SubmitJob(ctx, &awsbatch.SubmitJobInput{ + JobName: aws.String("x"), JobQueue: aws.String("q"), JobDefinition: aws.String("mnp"), + NodeOverrides: &batchtypes.NodeOverrides{NumNodes: aws.Int32(3)}, + }) + requireClientException(t, err, "SubmitJob nodeOverrides on a Fargate queue") + + _, err = c.SubmitJob(ctx, &awsbatch.SubmitJobInput{ + JobName: aws.String("x"), JobQueue: aws.String("q"), JobDefinition: aws.String("mnp"), + }) + requireClientException(t, err, "SubmitJob multinode definition on a Fargate queue") + + _, err = c.RegisterJobDefinition(ctx, &awsbatch.RegisterJobDefinitionInput{ + JobDefinitionName: aws.String("mnp-fg"), Type: batchtypes.JobDefinitionTypeMultinode, + PlatformCapabilities: []batchtypes.PlatformCapability{batchtypes.PlatformCapabilityFargate}, + NodeProperties: &batchtypes.NodeProperties{ + NumNodes: aws.Int32(2), MainNode: aws.Int32(0), + NodeRangeProperties: []batchtypes.NodeRangeProperty{ + {TargetNodes: aws.String("0:"), Container: &batchtypes.ContainerProperties{Image: aws.String("mpi")}}, + }, + }, + }) + requireClientException(t, err, "RegisterJobDefinition multinode with FARGATE") +}