From 27faf5c04e8caf49035a494ea8c251f8fbdd06e1 Mon Sep 17 00:00:00 2001 From: nisay759 Date: Thu, 30 Jul 2026 20:27:00 +0200 Subject: [PATCH 01/20] fix: data race in the /metrics handler The /metrics handler assigned h.options.Stats (a *Metrics) to a local variable, which copied the pointer rather than the struct. Every request therefore mutated the single shared Metrics struct when setting Cache, Cpu, Memory and Network, racing with concurrent /metrics requests, and read the counters non-atomically while the protocol servers updated them with atomic adds. Snapshot the counters into a local value using atomic loads instead, so the shared struct is never written and the encoded values are consistent. Add a regression test asserting the handler leaves the shared struct untouched, plus a concurrent metrics test that fails under -race on the old code. Co-Authored-By: Claude --- pkg/server/http_server.go | 18 ++++++- pkg/server/metrics_race_test.go | 88 +++++++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+), 2 deletions(-) create mode 100644 pkg/server/metrics_race_test.go diff --git a/pkg/server/http_server.go b/pkg/server/http_server.go index 7770e6b7..832278aa 100644 --- a/pkg/server/http_server.go +++ b/pkg/server/http_server.go @@ -425,6 +425,7 @@ func (h *HTTPServer) deregisterHandler(w http.ResponseWriter, req *http.Request) jsonError(w, fmt.Sprintf("could not remove id: %s", err), http.StatusBadRequest) return } + if h.options.RootTLD { for _, domain := range h.options.Domains { _ = h.options.Storage.RemoveConsumer(domain, r.CorrelationID) @@ -549,7 +550,20 @@ func (h *HTTPServer) checkToken(req *http.Request) bool { // metricsHandler is a handler for /metrics endpoint func (h *HTTPServer) metricsHandler(w http.ResponseWriter, req *http.Request) { - interactMetrics := h.options.Stats + // h.options.Stats is a *Metrics shared by every protocol server, whose + // counters are updated concurrently with atomic adds. Snapshot it into a + // local value with atomic loads rather than mutating the shared struct, + // which would race with those writers and with concurrent /metrics calls. + interactMetrics := Metrics{ + Dns: atomic.LoadUint64(&h.options.Stats.Dns), + Ftp: atomic.LoadUint64(&h.options.Stats.Ftp), + Http: atomic.LoadUint64(&h.options.Stats.Http), + Ldap: atomic.LoadUint64(&h.options.Stats.Ldap), + Smb: atomic.LoadUint64(&h.options.Stats.Smb), + Smtp: atomic.LoadUint64(&h.options.Stats.Smtp), + Sessions: atomic.LoadInt64(&h.options.Stats.Sessions), + SessionsTotal: atomic.LoadInt64(&h.options.Stats.SessionsTotal), + } interactMetrics.Cache = GetCacheMetrics(h.options) interactMetrics.Cpu = GetCpuMetrics() interactMetrics.Memory = GetMemoryMetrics() @@ -557,5 +571,5 @@ func (h *HTTPServer) metricsHandler(w http.ResponseWriter, req *http.Request) { w.Header().Set("Content-Type", "application/json; charset=utf-8") w.Header().Set("X-Content-Type-Options", "nosniff") - _ = json.NewEncoder(w).Encode(interactMetrics) + _ = json.NewEncoder(w).Encode(&interactMetrics) } diff --git a/pkg/server/metrics_race_test.go b/pkg/server/metrics_race_test.go new file mode 100644 index 00000000..114b0a00 --- /dev/null +++ b/pkg/server/metrics_race_test.go @@ -0,0 +1,88 @@ +package server + +import ( + "net/http" + "net/http/httptest" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/projectdiscovery/interactsh/pkg/storage" + "github.com/stretchr/testify/require" +) + +// newMetricsTestServer returns an HTTPServer backed by in-memory storage and a +// zeroed metrics struct, suitable for driving metricsHandler directly. +func newMetricsTestServer(t *testing.T) *HTTPServer { + t.Helper() + + store, err := storage.New(&storage.Options{EvictionTTL: 1 * time.Hour}) + require.NoError(t, err, "could not create storage") + t.Cleanup(func() { _ = store.Close() }) + + return &HTTPServer{options: &Options{Storage: store, Stats: &Metrics{}, EnableMetrics: true}} +} + +// TestMetricsHandlerDoesNotMutateSharedStats is a regression test for the +// /metrics handler aliasing options.Stats. It used to assign the *Metrics to a +// local variable, which copied the pointer rather than the struct, so every +// request wrote Cache/Cpu/Memory/Network into the one struct shared by all the +// protocol servers. The handler must leave that struct untouched. +func TestMetricsHandlerDoesNotMutateSharedStats(t *testing.T) { + h := newMetricsTestServer(t) + + w := httptest.NewRecorder() + h.metricsHandler(w, httptest.NewRequest("GET", "http://example.com/metrics", nil)) + require.Equal(t, http.StatusOK, w.Result().StatusCode) + + require.Nil(t, h.options.Stats.Cache, "handler must not write Cache into the shared stats") + require.Nil(t, h.options.Stats.Cpu, "handler must not write Cpu into the shared stats") + require.Nil(t, h.options.Stats.Memory, "handler must not write Memory into the shared stats") + require.Nil(t, h.options.Stats.Network, "handler must not write Network into the shared stats") +} + +// TestMetricsHandlerConcurrent exercises the /metrics snapshot against +// concurrent counter writers. Run with -race to catch regressions. +func TestMetricsHandlerConcurrent(t *testing.T) { + h := newMetricsTestServer(t) + + var wg sync.WaitGroup + stop := make(chan struct{}) + + // Writers mimic the protocol servers updating shared counters. + for i := 0; i < 4; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for { + select { + case <-stop: + return + default: + atomic.AddUint64(&h.options.Stats.Http, 1) + atomic.AddUint64(&h.options.Stats.Dns, 1) + atomic.AddInt64(&h.options.Stats.Sessions, 1) + atomic.AddInt64(&h.options.Stats.SessionsTotal, 1) + } + } + }() + } + + // Concurrent readers hitting the metrics endpoint. + for i := 0; i < 4; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < 50; j++ { + w := httptest.NewRecorder() + h.metricsHandler(w, httptest.NewRequest("GET", "http://example.com/metrics", nil)) + require.Equal(t, http.StatusOK, w.Result().StatusCode) + } + }() + } + + time.Sleep(100 * time.Millisecond) + close(stop) + wg.Wait() +} From 2d69507ee7c46db8bcb63b9d2bd92570072eabf5 Mon Sep 17 00:00:00 2001 From: nisay759 Date: Thu, 30 Jul 2026 23:54:31 +0200 Subject: [PATCH 02/20] storage: add upload metadata and eviction hook Groundwork for session-scoped file hosting. Uploaded file bytes will live on disk, but their lifecycle has to be driven by the correlation-id they belong to, so the metadata lives alongside the rest of the session state. - UploadedFile{Name,Size,SHA256,Timestamp} and a Files slice on CorrelationData, guarded by its existing mutex. Reusing CorrelationData rather than a second cache key means TTL expiry, capacity eviction, RemoveID and Close all cover the metadata with no second key to keep in sync. Not persisted: only interaction blobs go to leveldb, and uploads are not expected to survive a restart. - Options.OnEviction, invoked whenever a correlation-id leaves the cache, so the server can delete the corresponding files. This sits alongside the existing OnRemoval hook rather than replacing it: OnRemoval counts client sessions, while OnEviction fires for every entry and receives the evicted data, which is what file cleanup needs. - UpdateUploads verifies the secret key and runs a callback under the correlation-id lock. Callers do their disk write inside the callback, which makes the quota check and the commit atomic against concurrent uploads to the same session. ListUploads is the unauthenticated read path used when serving, and returns a copy. Both live on a separate UploadStorage interface rather than on Storage. Uploaded bytes are written to the local filesystem of one instance and the capacity quota is an in-process counter, so the capability is only coherent for instance-local backends. StorageDB implements it; the Redis backend deliberately does not, since a shared backend would let one instance advertise files whose bytes only exist on another instance's disk. Co-Authored-By: Claude --- pkg/storage/error.go | 6 + pkg/storage/option.go | 16 ++- pkg/storage/storage.go | 21 ++++ pkg/storage/storagedb.go | 59 ++++++++- pkg/storage/types.go | 14 +++ pkg/storage/uploads_test.go | 240 ++++++++++++++++++++++++++++++++++++ 6 files changed, 350 insertions(+), 6 deletions(-) create mode 100644 pkg/storage/uploads_test.go diff --git a/pkg/storage/error.go b/pkg/storage/error.go index b9a2aa22..23b97fd6 100644 --- a/pkg/storage/error.go +++ b/pkg/storage/error.go @@ -3,3 +3,9 @@ package storage import "github.com/projectdiscovery/utils/errkit" var ErrCorrelationIdNotFound = errkit.New("could not get correlation-id from cache") + +// ErrInvalidSecretKey is returned when the secret key presented for a +// correlation-id does not match the one stored at registration. It is +// distinguishable from ErrCorrelationIdNotFound so callers can tell "wrong +// session" apart from "no such session". +var ErrInvalidSecretKey = errkit.New("invalid secret key passed for correlation-id") diff --git a/pkg/storage/option.go b/pkg/storage/option.go index 1d91fb49..a72d6acd 100644 --- a/pkg/storage/option.go +++ b/pkg/storage/option.go @@ -10,14 +10,20 @@ const ( ) type Options struct { - DbPath string - EvictionTTL time.Duration - MaxSize int - MaxSharedInteractions int - EvictionStrategy EvictionStrategy + DbPath string + EvictionTTL time.Duration + MaxSize int + MaxSharedInteractions int + EvictionStrategy EvictionStrategy // OnRemoval is called for each client session removed from cache // (deregistration, TTL expiry, size eviction, or cache close). OnRemoval func() + // OnEviction is invoked when a correlation-id leaves the cache for any + // reason: explicit removal, TTL expiry, capacity eviction or Close. + // Unlike OnRemoval it fires for every entry, not just client sessions, and + // receives the evicted data so callers can release resources keyed off it. + // It runs on the cache's single event goroutine and must not block. + OnEviction func(correlationID string, data *CorrelationData) } func (options *Options) UseDisk() bool { diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 60e582b4..8b4885f6 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -15,3 +15,24 @@ type Storage interface { GetCacheItem(token string) (*CorrelationData, error) Close() error } + +// UploadStorage is the optional capability of tracking per-session uploaded +// file metadata, implemented by StorageDB. +// +// It is deliberately kept out of Storage. The uploaded bytes live on the local +// filesystem of a single server instance and the capacity quota is an +// in-process counter, so the feature is only coherent for instance-local +// backends. A shared backend such as Redis would let one instance advertise +// files whose bytes only exist on another instance's disk, so it does not +// implement this and file hosting is refused when it is selected. +type UploadStorage interface { + // UpdateUploads verifies the secret key for a correlation-id and then runs + // fn under that correlation-id's lock, replacing the upload metadata with + // whatever fn returns. Callers perform their disk writes inside fn so the + // quota check and the commit are atomic against concurrent uploads for the + // same session. fn is invoked at most once. + UpdateUploads(correlationID, secret string, fn func([]UploadedFile) ([]UploadedFile, error)) error + // ListUploads returns the upload metadata for a correlation-id, and whether + // the correlation-id is known at all. + ListUploads(correlationID string) ([]UploadedFile, bool) +} diff --git a/pkg/storage/storagedb.go b/pkg/storage/storagedb.go index 0bc65185..1cca55d1 100644 --- a/pkg/storage/storagedb.go +++ b/pkg/storage/storagedb.go @@ -83,13 +83,17 @@ func (s *StorageDB) onCacheRemoval(key cache.Key, value cache.Value) { if s.Options.UseDisk() && s.db != nil { _ = s.db.Delete([]byte(k), &opt.WriteOptions{}) } + cd, _ := value.(*CorrelationData) // Only fire for client sessions (entries with a SecretKey), // not for token/domain entries created via SetID. if s.Options.OnRemoval != nil { - if cd, ok := value.(*CorrelationData); ok && cd.SecretKey != "" { + if cd != nil && cd.SecretKey != "" { s.Options.OnRemoval() } } + if s.Options.OnEviction != nil { + s.Options.OnEviction(k, cd) + } } func (s *StorageDB) GetCacheMetrics() (*CacheMetrics, error) { @@ -461,6 +465,57 @@ func (s *StorageDB) RemoveID(correlationID, secret string) error { return nil } +// UpdateUploads verifies the secret key for a correlation-id and then runs fn +// while holding that correlation-id's lock, replacing the upload metadata with +// whatever fn returns. Callers perform their disk writes inside fn so that the +// quota check and the commit are atomic with respect to concurrent uploads +// against the same session. +func (s *StorageDB) UpdateUploads(correlationID, secret string, fn func([]UploadedFile) ([]UploadedFile, error)) error { + item, ok := s.cache.GetIfPresent(correlationID) + if !ok { + return ErrCorrelationIdNotFound + } + value, ok := item.(*CorrelationData) + if !ok { + return errors.New("invalid correlation-id cache value found") + } + if !strings.EqualFold(value.SecretKey, secret) { + return ErrInvalidSecretKey + } + value.Lock() + defer value.Unlock() + + updated, err := fn(value.Files) + if err != nil { + return err + } + value.Files = updated + return nil +} + +// ListUploads returns the upload metadata for a correlation-id. No secret is +// required: this backs the file-serving path, where possession of the +// correlation-id is the only credential. +func (s *StorageDB) ListUploads(correlationID string) ([]UploadedFile, bool) { + item, ok := s.cache.GetIfPresent(correlationID) + if !ok { + return nil, false + } + value, ok := item.(*CorrelationData) + if !ok { + return nil, false + } + value.Lock() + defer value.Unlock() + + if len(value.Files) == 0 { + return nil, true + } + files := make([]UploadedFile, len(value.Files)) + copy(files, value.Files) + return files, true +} + // GetCacheItem returns an item as is func (s *StorageDB) GetCacheItem(token string) (*CorrelationData, error) { item, ok := s.cache.GetIfPresent(token) @@ -529,3 +584,5 @@ func (s *StorageDB) Close() error { os.RemoveAll(s.dbpath), ) } + +var _ UploadStorage = (*StorageDB)(nil) diff --git a/pkg/storage/types.go b/pkg/storage/types.go index c7750cbc..13f1e485 100644 --- a/pkg/storage/types.go +++ b/pkg/storage/types.go @@ -16,6 +16,16 @@ type CacheMetrics struct { EvictionCount uint64 `json:"eviction-count"` } +// UploadedFile is metadata for a file uploaded by the owner of a correlation-id. +// The bytes themselves live on disk, managed by the server's upload store; this +// record exists so that cache eviction and deregistration can drive file cleanup. +type UploadedFile struct { + Name string `json:"name"` + Size int64 `json:"size"` + SHA256 string `json:"sha256"` + Timestamp time.Time `json:"timestamp"` +} + // CorrelationData is the data for a correlation-id. type CorrelationData struct { sync.Mutex @@ -29,4 +39,8 @@ type CorrelationData struct { AESKey []byte `json:"-"` ReadOffsets map[string]int `json:"-"` LastSeen map[string]time.Time `json:"-"` + // Files is metadata for files uploaded against this correlation-id. + // Guarded by the embedded Mutex. Not persisted: only interaction blobs + // are written to disk, and uploads do not survive a restart. + Files []UploadedFile `json:"-"` } diff --git a/pkg/storage/uploads_test.go b/pkg/storage/uploads_test.go new file mode 100644 index 00000000..9aff7e42 --- /dev/null +++ b/pkg/storage/uploads_test.go @@ -0,0 +1,240 @@ +package storage + +import ( + "os" + "sync" + "testing" + "time" + + "github.com/google/uuid" + "github.com/rs/xid" + "github.com/stretchr/testify/require" +) + +func newUploadTestDB(t *testing.T, opts *Options) (*StorageDB, string, string) { + t.Helper() + + db, err := New(opts) + require.NoError(t, err) + t.Cleanup(func() { _ = db.Close() }) + + secret := uuid.New().String() + correlationID := xid.New().String() + _, pubKey := generateRSAKeyPair(t) + require.NoError(t, db.SetIDPublicKey(correlationID, secret, pubKey)) + + return db, correlationID, secret +} + +func TestUpdateUploads(t *testing.T) { + t.Run("stores metadata", func(t *testing.T) { + db, id, secret := newUploadTestDB(t, &Options{EvictionTTL: time.Hour}) + + err := db.UpdateUploads(id, secret, func(existing []UploadedFile) ([]UploadedFile, error) { + require.Empty(t, existing, "new session should start with no uploads") + return append(existing, UploadedFile{Name: "evil.dtd", Size: 42}), nil + }) + require.NoError(t, err) + + files, ok := db.ListUploads(id) + require.True(t, ok) + require.Len(t, files, 1) + require.Equal(t, "evil.dtd", files[0].Name) + }) + + t.Run("rejects wrong secret", func(t *testing.T) { + db, id, _ := newUploadTestDB(t, &Options{EvictionTTL: time.Hour}) + + called := false + err := db.UpdateUploads(id, uuid.New().String(), func(f []UploadedFile) ([]UploadedFile, error) { + called = true + return f, nil + }) + require.ErrorIs(t, err, ErrInvalidSecretKey) + require.False(t, called, "callback must not run for an unauthorised caller") + }) + + t.Run("rejects unknown correlation id", func(t *testing.T) { + db, _, secret := newUploadTestDB(t, &Options{EvictionTTL: time.Hour}) + + err := db.UpdateUploads(xid.New().String(), secret, func(f []UploadedFile) ([]UploadedFile, error) { + return f, nil + }) + require.ErrorIs(t, err, ErrCorrelationIdNotFound) + }) + + t.Run("callback error leaves metadata untouched", func(t *testing.T) { + db, id, secret := newUploadTestDB(t, &Options{EvictionTTL: time.Hour}) + + require.NoError(t, db.UpdateUploads(id, secret, func(f []UploadedFile) ([]UploadedFile, error) { + return append(f, UploadedFile{Name: "keep.dtd"}), nil + })) + + err := db.UpdateUploads(id, secret, func(f []UploadedFile) ([]UploadedFile, error) { + return append(f, UploadedFile{Name: "discard.dtd"}), os.ErrInvalid + }) + require.ErrorIs(t, err, os.ErrInvalid) + + files, ok := db.ListUploads(id) + require.True(t, ok) + require.Len(t, files, 1, "failed update must not commit") + require.Equal(t, "keep.dtd", files[0].Name) + }) + + // The quota check and the commit both run inside the callback under the + // correlation-id lock, so concurrent uploads to one session cannot both + // observe the same "slots remaining" and overshoot. + t.Run("concurrent updates respect a quota", func(t *testing.T) { + db, id, secret := newUploadTestDB(t, &Options{EvictionTTL: time.Hour}) + + const maxFiles = 5 + var wg sync.WaitGroup + for i := 0; i < 25; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + _ = db.UpdateUploads(id, secret, func(f []UploadedFile) ([]UploadedFile, error) { + if len(f) >= maxFiles { + return nil, os.ErrPermission + } + return append(f, UploadedFile{Name: xid.New().String()}), nil + }) + }(i) + } + wg.Wait() + + files, ok := db.ListUploads(id) + require.True(t, ok) + require.Len(t, files, maxFiles, "quota must hold under concurrency") + }) +} + +func TestListUploads(t *testing.T) { + db, id, _ := newUploadTestDB(t, &Options{EvictionTTL: time.Hour}) + + t.Run("known id with no uploads", func(t *testing.T) { + files, ok := db.ListUploads(id) + require.True(t, ok, "session exists") + require.Empty(t, files) + }) + + t.Run("unknown id", func(t *testing.T) { + files, ok := db.ListUploads(xid.New().String()) + require.False(t, ok) + require.Nil(t, files) + }) + + t.Run("returns a copy", func(t *testing.T) { + db, id, secret := newUploadTestDB(t, &Options{EvictionTTL: time.Hour}) + require.NoError(t, db.UpdateUploads(id, secret, func(f []UploadedFile) ([]UploadedFile, error) { + return append(f, UploadedFile{Name: "orig.dtd"}), nil + })) + + files, _ := db.ListUploads(id) + files[0].Name = "mutated.dtd" + + again, _ := db.ListUploads(id) + require.Equal(t, "orig.dtd", again[0].Name, "caller must not be able to mutate stored metadata") + }) +} + +// TestOnEvictionMemoryMode covers OnEviction firing in memory mode, which is +// the default and the mode upload cleanup most depends on: with no leveldb +// handle in play, the hook is the only signal that a session's files may go. +func TestOnEvictionMemoryMode(t *testing.T) { + t.Run("fires on RemoveID", func(t *testing.T) { + var mu sync.Mutex + var evicted []string + + db, id, secret := newUploadTestDB(t, &Options{ + EvictionTTL: time.Hour, + OnEviction: func(correlationID string, _ *CorrelationData) { + mu.Lock() + defer mu.Unlock() + evicted = append(evicted, correlationID) + }, + }) + + require.NoError(t, db.RemoveID(id, secret)) + require.Eventually(t, func() bool { + mu.Lock() + defer mu.Unlock() + return len(evicted) == 1 && evicted[0] == id + }, 2*time.Second, 10*time.Millisecond) + }) + + t.Run("fires on ttl expiry", func(t *testing.T) { + var mu sync.Mutex + var evicted []string + + db, id, _ := newUploadTestDB(t, &Options{ + EvictionTTL: 100 * time.Millisecond, + OnEviction: func(correlationID string, _ *CorrelationData) { + mu.Lock() + defer mu.Unlock() + evicted = append(evicted, correlationID) + }, + }) + + time.Sleep(200 * time.Millisecond) + // goburrow/cache has no janitor; expiry is only processed on cache + // activity, so poke it. + db.cache.GetIfPresent(id) + + require.Eventually(t, func() bool { + mu.Lock() + defer mu.Unlock() + return len(evicted) == 1 && evicted[0] == id + }, 2*time.Second, 10*time.Millisecond) + }) + + t.Run("fires on close", func(t *testing.T) { + var mu sync.Mutex + var evicted []string + + db, err := New(&Options{ + EvictionTTL: time.Hour, + OnEviction: func(correlationID string, _ *CorrelationData) { + mu.Lock() + defer mu.Unlock() + evicted = append(evicted, correlationID) + }, + }) + require.NoError(t, err) + + secret := uuid.New().String() + id := xid.New().String() + _, pubKey := generateRSAKeyPair(t) + require.NoError(t, db.SetIDPublicKey(id, secret, pubKey)) + + require.NoError(t, db.Close()) + require.Eventually(t, func() bool { + mu.Lock() + defer mu.Unlock() + return len(evicted) == 1 && evicted[0] == id + }, 2*time.Second, 10*time.Millisecond) + }) + + // Without the s.db nil guard this panics on the cache's event goroutine, + // which then deadlocks every subsequent cache operation. + t.Run("memory mode eviction does not panic", func(t *testing.T) { + db, id, secret := newUploadTestDB(t, &Options{EvictionTTL: time.Hour}) + + require.NoError(t, db.RemoveID(id, secret)) + time.Sleep(100 * time.Millisecond) + + // If the event goroutine had died, this would block forever. + done := make(chan struct{}) + go func() { + defer close(done) + newID := xid.New().String() + _, pubKey := generateRSAKeyPair(t) + _ = db.SetIDPublicKey(newID, uuid.New().String(), pubKey) + }() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("cache is deadlocked: the removal listener killed the event goroutine") + } + }) +} From 02f4e6bddedc597fc5d1fc4d37da5050dd4ea88c Mon Sep 17 00:00:00 2001 From: nisay759 Date: Fri, 31 Jul 2026 00:01:43 +0200 Subject: [PATCH 03/20] server: add UploadStore for session-scoped file hosting Owns the on-disk lifecycle of client-uploaded files, laid out as /.interactsh-user-uploads//. The session directory is named by correlation ID because FTP has no Host header, so the identifier has to travel in the path for the FTP view to resolve it. Sessions sit under .interactsh-user-uploads rather than directly at the root so that the root can be shared with -ftp-dir safely. Everything this store creates, enumerates and deletes lives under that one directory, so an operator directory that happens to share the correlation-ID name shape can never be reaped -- and that shape is not hard to collide with, since -cidl goes as low as 3. Root resolution prefers -upload-directory, then falls back to the FTP root (so -ftp serves uploads with no extra configuration), then to a temporary directory. Creating the sessions directory creates the root with it, so a -ftp-dir that does not exist yet -- the FTP server never required it to -- does not turn into a boot failure when -upload is added. Anything already inside the sessions directory is purged at startup: upload metadata lives only in the cache, so no session survives a restart and any surviving directory is an orphan. The purge and the janitor both skip entries that do not look like a correlation ID, which is a second line of defence for anything unexpected in there; operator files are kept safe structurally, by being outside that directory entirely. File names go through a strict allowlist rather than a sanitiser. The user needs the exact byte-for-byte name to reference the file from a DTD, so mangling it silently is worse than rejecting it; it also means the name needs no escaping in a Content-Disposition header. Reads resolve through an os.Root handle, which refuses to escape the root or follow a symlink out of it on every platform we build for. Writes go to a temp name and are renamed into place, so neither the HTTP handler nor the FTP file driver can observe a partially written file. Cleanup runs three ways, because none of them is sufficient alone: - RemoveSession, queued to a deleter goroutine via a non-blocking send. It is called from the storage cache's event goroutine, where a synchronous RemoveAll would serialise all cache maintenance behind a 64-slot channel. - A mtime-based janitor, which is the authoritative collector. goburrow/cache has no background janitor, so an idle server never evicts and would otherwise leak every uploaded file indefinitely. - The startup purge above. The janitor deliberately judges liveness by directory mtime and never asks the cache: GetIfPresent refreshes access time, so probing each swept session would make exactly those sessions immortal under sliding eviction. Co-Authored-By: Claude The upload directory is probed for writability before the store is returned. MkdirAll reports success for a directory that already exists but cannot be written to -- a read-only mount, or one owned by another user -- so without the probe that misconfiguration would first surface as a 500 on a client's initial upload, long after startup. The probe creates, writes and removes a file rather than reading permission bits, which answer for the wrong subject on a setuid binary and say nothing at all about a read-only mount, an exhausted filesystem or a restrictive ACL. --- pkg/server/server.go | 15 ++ pkg/server/upload.go | 490 ++++++++++++++++++++++++++++++++++++++ pkg/server/upload_test.go | 363 ++++++++++++++++++++++++++++ 3 files changed, 868 insertions(+) create mode 100644 pkg/server/upload.go create mode 100644 pkg/server/upload_test.go diff --git a/pkg/server/server.go b/pkg/server/server.go index c480e37b..83981776 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -111,6 +111,21 @@ type Options struct { // DefaultHTTPResponseFile is a file to serve for all HTTP requests (takes priority over other options) DefaultHTTPResponseFile string + // Upload enables the client file upload endpoint and file hosting + Upload bool + // UploadDirectory is the root directory uploaded files are stored under + UploadDirectory string + // UploadMaxFileSize is the maximum size in bytes of a single uploaded file + UploadMaxFileSize int64 + // UploadMaxFiles is the maximum number of files a single session may upload + UploadMaxFiles int + // UploadMaxTotalSize is the maximum total size in bytes of all uploaded files + UploadMaxTotalSize int64 + // UploadTTL is the maximum lifetime of uploaded files on disk + UploadTTL time.Duration + // UploadStore serves and stores uploaded files. Nil when uploads are disabled. + UploadStore *UploadStore + ACMEStore *acme.Provider Stats *Metrics OnResult OnResultCallback diff --git a/pkg/server/upload.go b/pkg/server/upload.go new file mode 100644 index 00000000..4d0e8c8e --- /dev/null +++ b/pkg/server/upload.go @@ -0,0 +1,490 @@ +package server + +import ( + "crypto/sha256" + "encoding/hex" + "os" + "path/filepath" + "regexp" + "sync" + "sync/atomic" + "time" + + "github.com/asaskevich/govalidator" + "github.com/pkg/errors" + "github.com/projectdiscovery/gologger" + "github.com/rs/xid" +) + +const ( + uploadSessionDirPerm = 0o700 + uploadFilePerm = 0o600 + + // uploadsDirName is the single directory, directly under the upload root, + // that holds every session's uploaded files. + // + // Sessions live one level down rather than at the root so that the root can + // be a shared -ftp-dir full of the operator's own files: the janitor and the + // startup purge only ever read, and delete, inside this directory, so no + // amount of coincidence between an operator's directory name and a + // correlation id can put their content at risk. The leading dot keeps it out + // of the way of an operator listing their own FTP root; it is not a security + // measure, since NopDriver is what actually hides it (see ftp_server.go). + uploadsDirName = ".interactsh-user-uploads" + + // deleteQueueSize bounds the backlog of session directories awaiting + // removal. Sends are non-blocking, so a full queue drops the request and + // leaves the directory for the janitor. + deleteQueueSize = 1024 + + // maxSweepInterval caps how long the janitor sleeps regardless of TTL. + maxSweepInterval = 10 * time.Minute + minSweepInterval = 1 * time.Minute +) + +// uploadNameRe is a strict allowlist for uploaded file names. It is applied +// identically on upload and on serve. +// +// This is deliberately an allowlist rather than a sanitiser: the user needs the +// exact byte-for-byte name to reference the file from a DTD or XSLT payload, so +// silently mangling a name is worse than rejecting it. It also means the name +// never needs escaping when it is placed in a Content-Disposition header. +// +// Rejects: path separators, "..", NUL and control characters, leading dots, +// absolute paths, and over-long names. +var uploadNameRe = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$`) + +// isSafeUploadName reports whether name is acceptable as an uploaded file name. +func isSafeUploadName(name string) bool { + if !uploadNameRe.MatchString(name) { + return false + } + // Belt and braces: the regexp already excludes "/" and "\", so no traversal + // sequence can survive, but a bare ".." must never slip through either. + return name != "." && name != ".." +} + +// UploadErrorKind classifies upload failures so the HTTP layer can map them to +// status codes without matching on error strings. +type UploadErrorKind int + +const ( + UploadErrOther UploadErrorKind = iota + UploadErrBadName + UploadErrTooLarge + UploadErrTooManyFiles + UploadErrOutOfSpace +) + +// UploadError is a failure attributable to the uploaded content or to server +// capacity, as opposed to an internal error. +type UploadError struct { + Kind UploadErrorKind + Err error +} + +func (e *UploadError) Error() string { return e.Err.Error() } +func (e *UploadError) Unwrap() error { return e.Err } + +func uploadErr(kind UploadErrorKind, format string, args ...interface{}) *UploadError { + return &UploadError{Kind: kind, Err: errors.Errorf(format, args...)} +} + +// UploadStore owns the on-disk lifecycle of client-uploaded files. +// +// Layout is ///. The session +// directory is named by correlation ID because FTP has no Host header, so the +// identifier has to be carried in the path for the FTP view to work. The +// uploadsDirName level exists so that the root may be shared with -ftp-dir +// without this store ever touching an operator's files. +type UploadStore struct { + root string + // sessionsRoot is /: everything this store creates, + // enumerates and deletes lives under it, and nothing above it is ours. + sessionsRoot string + // rootFS is an openat-rooted handle used for the serving path. Names there + // come from a URL, and os.Root refuses any resolution that escapes the root + // or traverses a symlink out of it, on every platform we build for. + rootFS *os.Root + maxFileSize int64 + maxFiles int + maxTotal int64 + ttl time.Duration + + // correlationIDLength is used to recognise session directories inside + // sessionsRoot, and to reject implausible identifiers before they reach a + // filepath.Join. + correlationIDLength int + + totalBytes atomic.Int64 + + deleteCh chan string + closeCh chan struct{} + closeWG sync.WaitGroup + closeAt sync.Once +} + +// NewUploadStore prepares the upload root and returns a store. It does not +// start the background goroutines; call Start for that. +func NewUploadStore(options *Options) (*UploadStore, error) { + root := options.UploadDirectory + switch { + case root != "": + case options.FTPDirectory != "": + // Share the FTP root so that -ftp serves uploads with no extra config. + root = options.FTPDirectory + default: + var err error + if root, err = os.MkdirTemp("", "interactsh-uploads-"); err != nil { + return nil, errors.Wrap(err, "could not create temporary upload directory") + } + } + + abs, err := filepath.Abs(root) + if err != nil { + return nil, errors.Wrap(err, "could not resolve upload directory") + } + + // Creating the sessions directory creates the root along with it, so every + // branch above ends up with both present before os.OpenRoot needs them -- + // including a -ftp-dir that does not exist yet, which the FTP server itself + // never required. + sessionsRoot := filepath.Join(abs, uploadsDirName) + if err := os.MkdirAll(sessionsRoot, uploadSessionDirPerm); err != nil { + return nil, errors.Wrap(err, "could not create upload directory") + } + + // MkdirAll succeeds on a directory that already exists but cannot be written + // to -- a read-only mount, or one owned by another user -- and the failure + // would otherwise surface only when a client's first upload 500s, long after + // startup. Probe it now so the server refuses to start with a clear reason. + if err := checkUploadDirWritable(sessionsRoot); err != nil { + return nil, err + } + + rootFS, err := os.OpenRoot(abs) + if err != nil { + return nil, errors.Wrap(err, "could not open upload directory") + } + + s := &UploadStore{ + root: abs, + sessionsRoot: sessionsRoot, + rootFS: rootFS, + maxFileSize: options.UploadMaxFileSize, + maxFiles: options.UploadMaxFiles, + maxTotal: options.UploadMaxTotalSize, + ttl: options.UploadTTL, + correlationIDLength: options.CorrelationIdLength, + deleteCh: make(chan string, deleteQueueSize), + closeCh: make(chan struct{}), + } + + // Upload metadata lives only in the cache, so no session survives a + // restart; anything already here is by definition an orphan. + s.purge() + return s, nil +} + +// checkUploadDirWritable verifies that files can actually be created in dir, by +// doing it. Permission bits are not consulted directly: they answer the question +// for the wrong subject on a setuid binary, and they do not answer it at all for +// a read-only mount, an exhausted filesystem or a restrictive ACL. +func checkUploadDirWritable(dir string) error { + f, err := os.CreateTemp(dir, ".writable-*") + if err != nil { + return errors.Wrapf(err, "upload directory %s is not writable", dir) + } + name := f.Name() + defer func() { _ = os.Remove(name) }() + + // Written to as well as created, so that a filesystem which allows the + // create but refuses the write is caught here rather than mid-upload. + if _, err := f.Write([]byte("interactsh")); err != nil { + _ = f.Close() + return errors.Wrapf(err, "upload directory %s is not writable", dir) + } + if err := f.Close(); err != nil { + return errors.Wrapf(err, "upload directory %s is not writable", dir) + } + return nil +} + +// Root returns the directory uploaded files are stored under. +func (s *UploadStore) Root() string { return s.root } + +// MaxFileSize returns the per-file byte limit. +func (s *UploadStore) MaxFileSize() int64 { return s.maxFileSize } + +// MaxFiles returns the per-session file count limit. +func (s *UploadStore) MaxFiles() int { return s.maxFiles } + +// Start launches the deleter and janitor goroutines. +func (s *UploadStore) Start() { + s.closeWG.Add(2) + go s.runDeleter() + go s.runJanitor() +} + +// Close stops the background goroutines and releases the rooted handle. +func (s *UploadStore) Close() error { + s.closeAt.Do(func() { close(s.closeCh) }) + s.closeWG.Wait() + return s.rootFS.Close() +} + +// looksLikeSessionDir reports whether a directory entry name could be one of +// our session directories. +// +// Operator files are kept safe structurally, by everything of ours living under +// uploadsDirName; this check is the second line of defence, so that anything +// unexpected inside that directory is left alone rather than deleted. +func (s *UploadStore) looksLikeSessionDir(name string) bool { + return len(name) == s.correlationIDLength && govalidator.IsAlphanumeric(name) +} + +// sessionDir returns the directory for a correlation ID, or "" if the ID is not +// a plausible correlation ID. Validating before any filepath.Join is what keeps +// a hostile identifier from escaping the root. +func (s *UploadStore) sessionDir(correlationID string) string { + if !s.looksLikeSessionDir(correlationID) { + return "" + } + return filepath.Join(s.sessionsRoot, correlationID) +} + +// Save writes one file for a correlation ID and returns its size and digest. +// +// It is called from inside Storage.UpdateUploads, i.e. under the correlation +// ID's lock, so the caller's quota check and this write are atomic with respect +// to other uploads for the same session. +func (s *UploadStore) Save(correlationID, name string, data []byte, existingSize int64) (int64, string, error) { + if !isSafeUploadName(name) { + return 0, "", uploadErr(UploadErrBadName, "invalid file name %q", name) + } + size := int64(len(data)) + if size == 0 { + return 0, "", uploadErr(UploadErrBadName, "file %q is empty", name) + } + if size > s.maxFileSize { + return 0, "", uploadErr(UploadErrTooLarge, "file %q is %d bytes, limit is %d", name, size, s.maxFileSize) + } + + dir := s.sessionDir(correlationID) + if dir == "" { + return 0, "", uploadErr(UploadErrOther, "invalid correlation-id") + } + + // Reserve space before writing. existingSize is what this name already + // occupies, since overwriting replaces rather than adds. + if delta := size - existingSize; delta > 0 { + if s.totalBytes.Add(delta) > s.maxTotal { + s.totalBytes.Add(-delta) + return 0, "", uploadErr(UploadErrOutOfSpace, "server upload capacity exhausted") + } + } else { + s.totalBytes.Add(delta) + } + + if err := os.MkdirAll(dir, uploadSessionDirPerm); err != nil { + s.totalBytes.Add(existingSize - size) + return 0, "", errors.Wrap(err, "could not create session directory") + } + + // Plain os rather than the rooted handle: os.Root has no Rename before Go + // 1.25, and both components are already constrained -- the correlation ID + // is alphanumeric and length-checked, the name passed the allowlist, so + // neither can contain a separator or traversal sequence. + // + // Write to a temp name and rename into place, so a reader (HTTP or the FTP + // file driver) never observes a partially written file. + tmp := filepath.Join(dir, ".upload-"+xid.New().String()) + if err := os.WriteFile(tmp, data, uploadFilePerm); err != nil { + s.totalBytes.Add(existingSize - size) + return 0, "", errors.Wrap(err, "could not write uploaded file") + } + if err := os.Rename(tmp, filepath.Join(dir, name)); err != nil { + _ = os.Remove(tmp) + s.totalBytes.Add(existingSize - size) + return 0, "", errors.Wrap(err, "could not commit uploaded file") + } + + sum := sha256.Sum256(data) + return size, hex.EncodeToString(sum[:]), nil +} + +// Open returns a readable handle to an uploaded file. The name is re-validated +// here rather than trusted from the caller. +func (s *UploadStore) Open(correlationID, name string) (*os.File, os.FileInfo, error) { + if !isSafeUploadName(name) { + return nil, nil, errors.New("invalid file name") + } + if !s.looksLikeSessionDir(correlationID) { + return nil, nil, errors.New("invalid correlation-id") + } + + // Resolved through the rooted handle, so neither component can escape the + // upload directory even if the root is operator-supplied and contains a + // planted symlink. + f, err := s.rootFS.Open(filepath.Join(uploadsDirName, correlationID, name)) + if err != nil { + return nil, nil, err + } + fi, err := f.Stat() + if err != nil { + _ = f.Close() + return nil, nil, err + } + if !fi.Mode().IsRegular() { + _ = f.Close() + return nil, nil, errors.New("not a regular file") + } + return f, fi, nil +} + +// RemoveSession queues a session's directory for deletion. It is safe to call +// from the storage cache's event goroutine: the send is non-blocking, so +// filesystem latency can never back-pressure cache maintenance. A dropped +// request is collected by the janitor instead. +func (s *UploadStore) RemoveSession(correlationID string) { + if s.sessionDir(correlationID) == "" { + return + } + select { + case s.deleteCh <- correlationID: + default: + gologger.Debug().Msgf("Upload delete queue full, leaving %s to the janitor\n", correlationID) + } +} + +// removeSessionNow deletes a session directory synchronously. +func (s *UploadStore) removeSessionNow(correlationID string) { + dir := s.sessionDir(correlationID) + if dir == "" { + return + } + freed := dirSize(dir) + if err := os.RemoveAll(dir); err != nil { + gologger.Warning().Msgf("Could not remove upload directory for %s: %s\n", correlationID, err) + return + } + if freed > 0 { + s.totalBytes.Add(-freed) + } +} + +func (s *UploadStore) runDeleter() { + defer s.closeWG.Done() + for { + select { + case id := <-s.deleteCh: + s.removeSessionNow(id) + case <-s.closeCh: + // Drain whatever is already queued, then stop. + for { + select { + case id := <-s.deleteCh: + s.removeSessionNow(id) + default: + return + } + } + } + } +} + +// runJanitor is the authoritative garbage collector for uploaded files. +// +// It cannot be left to cache eviction: goburrow/cache has no background +// janitor, so expiry is only processed on cache activity. An idle server would +// never evict, and would therefore leak every uploaded file indefinitely. +func (s *UploadStore) runJanitor() { + defer s.closeWG.Done() + + interval := s.ttl / 10 + if interval > maxSweepInterval { + interval = maxSweepInterval + } + if interval < minSweepInterval { + interval = minSweepInterval + } + + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ticker.C: + s.sweep() + case <-s.closeCh: + return + } + } +} + +// sweep removes session directories older than the TTL and recomputes the +// global byte total. +// +// Liveness is judged by directory mtime, never by asking the cache. A cache +// lookup would refresh the entry's access time, so under the default sliding +// eviction strategy probing every swept session would make exactly those +// sessions immortal. +func (s *UploadStore) sweep() { + entries, err := os.ReadDir(s.sessionsRoot) + if err != nil { + gologger.Warning().Msgf("Could not read upload directory: %s\n", err) + return + } + + var total int64 + for _, entry := range entries { + if !entry.IsDir() || !s.looksLikeSessionDir(entry.Name()) { + continue + } + dir := filepath.Join(s.sessionsRoot, entry.Name()) + info, err := entry.Info() + if err != nil { + continue + } + // Directory mtime advances when a file is added or removed, so this is + // effectively "time since the last upload". Reads do not touch it. + if time.Since(info.ModTime()) > s.ttl { + if err := os.RemoveAll(dir); err != nil { + gologger.Warning().Msgf("Could not sweep upload directory %s: %s\n", dir, err) + total += dirSize(dir) + } + continue + } + total += dirSize(dir) + } + s.totalBytes.Store(total) +} + +// purge removes every session directory in the root, leaving anything that does +// not look like one untouched. +func (s *UploadStore) purge() { + entries, err := os.ReadDir(s.sessionsRoot) + if err != nil { + return + } + for _, entry := range entries { + if entry.IsDir() && s.looksLikeSessionDir(entry.Name()) { + _ = os.RemoveAll(filepath.Join(s.sessionsRoot, entry.Name())) + } + } + s.totalBytes.Store(0) +} + +// dirSize sums the regular files directly inside dir. +func dirSize(dir string) int64 { + entries, err := os.ReadDir(dir) + if err != nil { + return 0 + } + var total int64 + for _, entry := range entries { + if info, err := entry.Info(); err == nil && info.Mode().IsRegular() { + total += info.Size() + } + } + return total +} diff --git a/pkg/server/upload_test.go b/pkg/server/upload_test.go new file mode 100644 index 00000000..99bd66b6 --- /dev/null +++ b/pkg/server/upload_test.go @@ -0,0 +1,363 @@ +package server + +import ( + "errors" + "io" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/projectdiscovery/interactsh/pkg/settings" + "github.com/rs/xid" + "github.com/stretchr/testify/require" +) + +func TestIsSafeUploadName(t *testing.T) { + valid := []string{ + "evil.dtd", "a", "a-b_c.1", "payload.xml", "x.tar.gz", + strings.Repeat("a", 128), + } + for _, name := range valid { + require.True(t, isSafeUploadName(name), "expected %q to be accepted", name) + } + + invalid := []string{ + "", ".", "..", "...", + "../evil", "../../etc/passwd", "a/b", "/etc/passwd", + `..\evil`, `a\b`, `C:\evil`, + ".hidden", "-leading-dash", + "evil\x00.dtd", "evil\n.dtd", "evil\r.dtd", "tab\t.dtd", + "space name.dtd", `quote".dtd`, "semi;colon", "pipe|d", + "percent%2e%2e", "unicodeé.dtd", "emoji\U0001f600", + strings.Repeat("a", 129), + } + for _, name := range invalid { + require.False(t, isSafeUploadName(name), "expected %q to be rejected", name) + } +} + +func newTestUploadStore(t *testing.T, tune func(*Options)) *UploadStore { + t.Helper() + + dir := t.TempDir() + options := &Options{ + CorrelationIdLength: settings.CorrelationIdLengthDefault, + UploadDirectory: dir, + UploadMaxFileSize: 1024, + UploadMaxFiles: 5, + UploadMaxTotalSize: 1 << 20, + UploadTTL: time.Hour, + } + if tune != nil { + tune(options) + } + + store, err := NewUploadStore(options) + require.NoError(t, err) + t.Cleanup(func() { _ = store.Close() }) + return store +} + +// newCorrelationID returns an id of the default configured length. +func newCorrelationID(t *testing.T) string { + t.Helper() + id := xid.New().String() + strings.Repeat("z", settings.CorrelationIdLengthDefault) + return id[:settings.CorrelationIdLengthDefault] +} + +func TestUploadStoreSaveOpen(t *testing.T) { + store := newTestUploadStore(t, nil) + id := newCorrelationID(t) + + t.Run("round trip", func(t *testing.T) { + content := []byte(``) + size, sum, err := store.Save(id, "evil.dtd", content, 0) + require.NoError(t, err) + require.EqualValues(t, len(content), size) + require.Len(t, sum, 64, "sha256 hex digest") + + f, fi, err := store.Open(id, "evil.dtd") + require.NoError(t, err) + defer f.Close() + + got, err := io.ReadAll(f) + require.NoError(t, err) + require.Equal(t, content, got) + require.EqualValues(t, len(content), fi.Size()) + }) + + t.Run("no temp files left visible", func(t *testing.T) { + entries, err := os.ReadDir(filepath.Join(store.sessionsRoot, id)) + require.NoError(t, err) + for _, e := range entries { + require.False(t, strings.HasPrefix(e.Name(), ".upload-"), + "partial write %q should have been renamed into place", e.Name()) + } + }) + + t.Run("rejects unsafe name", func(t *testing.T) { + _, _, err := store.Save(id, "../escape", []byte("x"), 0) + require.Error(t, err) + var ue *UploadError + require.True(t, errors.As(err, &ue)) + require.Equal(t, UploadErrBadName, ue.Kind) + }) + + t.Run("rejects empty file", func(t *testing.T) { + _, _, err := store.Save(id, "empty.dtd", nil, 0) + require.Error(t, err) + }) + + t.Run("rejects oversize file", func(t *testing.T) { + _, _, err := store.Save(id, "big.dtd", make([]byte, 2048), 0) + require.Error(t, err) + var ue *UploadError + require.True(t, errors.As(err, &ue)) + require.Equal(t, UploadErrTooLarge, ue.Kind) + }) + + t.Run("rejects invalid correlation id", func(t *testing.T) { + _, _, err := store.Save("../..", "x.dtd", []byte("x"), 0) + require.Error(t, err) + }) + + t.Run("open rejects traversal", func(t *testing.T) { + _, _, err := store.Open(id, "../../etc/passwd") + require.Error(t, err) + }) + + t.Run("open rejects unknown file", func(t *testing.T) { + _, _, err := store.Open(id, "absent.dtd") + require.Error(t, err) + }) +} + +func TestUploadStoreGlobalQuota(t *testing.T) { + store := newTestUploadStore(t, func(o *Options) { + o.UploadMaxFileSize = 1024 + o.UploadMaxTotalSize = 2048 + }) + + id := newCorrelationID(t) + _, _, err := store.Save(id, "a.bin", make([]byte, 1024), 0) + require.NoError(t, err) + _, _, err = store.Save(id, "b.bin", make([]byte, 1024), 0) + require.NoError(t, err) + + _, _, err = store.Save(id, "c.bin", make([]byte, 1024), 0) + require.Error(t, err, "third file should exhaust the global cap") + var ue *UploadError + require.True(t, errors.As(err, &ue)) + require.Equal(t, UploadErrOutOfSpace, ue.Kind) + + // Overwriting an existing name must account for the space it already holds + // rather than double-counting it. + _, _, err = store.Save(id, "a.bin", make([]byte, 1024), 1024) + require.NoError(t, err, "replacing a file of equal size must not exceed the cap") +} + +func TestUploadStoreRemoveSession(t *testing.T) { + store := newTestUploadStore(t, nil) + store.Start() + + id := newCorrelationID(t) + _, _, err := store.Save(id, "evil.dtd", []byte("payload"), 0) + require.NoError(t, err) + require.DirExists(t, filepath.Join(store.sessionsRoot, id)) + + store.RemoveSession(id) + require.Eventually(t, func() bool { + _, err := os.Stat(filepath.Join(store.sessionsRoot, id)) + return os.IsNotExist(err) + }, 2*time.Second, 10*time.Millisecond) + + // Idempotent: a second removal of a gone session must not error or panic. + store.RemoveSession(id) + store.RemoveSession(newCorrelationID(t)) + time.Sleep(50 * time.Millisecond) +} + +func TestUploadStoreSweep(t *testing.T) { + store := newTestUploadStore(t, func(o *Options) { o.UploadTTL = time.Hour }) + + stale := newCorrelationID(t) + fresh := newCorrelationID(t) + _, _, err := store.Save(stale, "old.dtd", []byte("old"), 0) + require.NoError(t, err) + _, _, err = store.Save(fresh, "new.dtd", []byte("new"), 0) + require.NoError(t, err) + + // Operator files sharing the root (the -ftp-dir case) must survive, including + // a directory whose name happens to look exactly like a correlation id: the + // sweeper only ever descends into its own directory, so a collision in the + // operator's namespace cannot cost them data. + sibling := filepath.Join(store.Root(), "operator-notes.txt") + require.NoError(t, os.WriteFile(sibling, []byte("keep me"), 0o600)) + siblingDir := filepath.Join(store.Root(), "operator-dir") + require.NoError(t, os.Mkdir(siblingDir, 0o700)) + require.NoError(t, os.Chtimes(siblingDir, time.Now().Add(-48*time.Hour), time.Now().Add(-48*time.Hour))) + lookalike := filepath.Join(store.Root(), newCorrelationID(t)) + require.NoError(t, os.Mkdir(lookalike, 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(lookalike, "payload.txt"), []byte("operator payload"), 0o600)) + require.NoError(t, os.Chtimes(lookalike, time.Now().Add(-48*time.Hour), time.Now().Add(-48*time.Hour))) + + old := time.Now().Add(-2 * time.Hour) + require.NoError(t, os.Chtimes(filepath.Join(store.sessionsRoot, stale), old, old)) + + store.sweep() + + require.NoDirExists(t, filepath.Join(store.sessionsRoot, stale), "expired session should be swept") + require.DirExists(t, filepath.Join(store.sessionsRoot, fresh), "live session should survive") + require.FileExists(t, sibling, "non-session file must not be swept") + require.DirExists(t, siblingDir, "non-session directory must not be swept even when old") + require.FileExists(t, filepath.Join(lookalike, "payload.txt"), + "an operator directory shaped like a correlation id must survive") +} + +func TestUploadStorePurgeOnStart(t *testing.T) { + dir := t.TempDir() + + orphan := newCorrelationID(t) + require.NoError(t, os.MkdirAll(filepath.Join(dir, uploadsDirName, orphan), 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(dir, uploadsDirName, orphan, "stale.dtd"), []byte("x"), 0o600)) + + keepFile := filepath.Join(dir, "index.html") + require.NoError(t, os.WriteFile(keepFile, []byte("operator content"), 0o600)) + keepDir := filepath.Join(dir, "assets") + require.NoError(t, os.Mkdir(keepDir, 0o700)) + // The startup purge is name-shape driven, so an operator directory that + // matches that shape has to be out of its reach structurally. + lookalike := filepath.Join(dir, newCorrelationID(t)) + require.NoError(t, os.Mkdir(lookalike, 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(lookalike, "payload.txt"), []byte("operator payload"), 0o600)) + + store := newTestUploadStore(t, func(o *Options) { o.UploadDirectory = dir }) + + require.NoDirExists(t, filepath.Join(dir, uploadsDirName, orphan), "orphaned session dir should be purged at startup") + require.FileExists(t, keepFile, "operator file must survive startup purge") + require.DirExists(t, keepDir, "operator directory must survive startup purge") + require.FileExists(t, filepath.Join(lookalike, "payload.txt"), + "an operator directory shaped like a correlation id must survive the purge") + _ = store +} + +func TestUploadStoreRootResolution(t *testing.T) { + t.Run("prefers upload directory", func(t *testing.T) { + up, ftp := t.TempDir(), t.TempDir() + store, err := NewUploadStore(&Options{ + CorrelationIdLength: settings.CorrelationIdLengthDefault, + UploadDirectory: up, FTPDirectory: ftp, UploadTTL: time.Hour, + }) + require.NoError(t, err) + defer store.Close() + require.Equal(t, up, store.Root()) + }) + + t.Run("falls back to ftp directory", func(t *testing.T) { + ftp := t.TempDir() + store, err := NewUploadStore(&Options{ + CorrelationIdLength: settings.CorrelationIdLengthDefault, + FTPDirectory: ftp, UploadTTL: time.Hour, + }) + require.NoError(t, err) + defer store.Close() + require.Equal(t, ftp, store.Root(), "sharing the FTP root is what makes FTP serving work") + }) + + t.Run("creates a shared ftp directory that does not exist yet", func(t *testing.T) { + // -ftp alone never required -ftp-dir to exist, so adopting it as the + // upload root must not turn a working deployment into a boot failure. + ftp := filepath.Join(t.TempDir(), "not-created-yet") + store, err := NewUploadStore(&Options{ + CorrelationIdLength: settings.CorrelationIdLengthDefault, + FTPDirectory: ftp, UploadTTL: time.Hour, + }) + require.NoError(t, err) + defer store.Close() + require.DirExists(t, ftp) + require.DirExists(t, filepath.Join(ftp, uploadsDirName)) + }) + + t.Run("sessions live under the uploads directory", func(t *testing.T) { + up := t.TempDir() + store, err := NewUploadStore(&Options{ + CorrelationIdLength: settings.CorrelationIdLengthDefault, + UploadDirectory: up, UploadTTL: time.Hour, UploadMaxFileSize: 1024, UploadMaxTotalSize: 4096, + }) + require.NoError(t, err) + defer store.Close() + + id := newCorrelationID(t) + _, _, err = store.Save(id, "evil.dtd", []byte("payload"), 0) + require.NoError(t, err) + require.FileExists(t, filepath.Join(up, uploadsDirName, id, "evil.dtd")) + require.NoDirExists(t, filepath.Join(up, id), "nothing of ours belongs at the root") + }) + + t.Run("falls back to a temp directory", func(t *testing.T) { + store, err := NewUploadStore(&Options{ + CorrelationIdLength: settings.CorrelationIdLengthDefault, + UploadTTL: time.Hour, + }) + require.NoError(t, err) + defer func() { + root := store.Root() + _ = store.Close() + _ = os.RemoveAll(root) + }() + require.DirExists(t, store.Root()) + }) +} + +// A directory can exist and still be unwritable, in which case MkdirAll succeeds +// and every upload fails later. The server must refuse to start instead. +func TestUploadStoreRejectsUnwritableDirectory(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("running as root bypasses the permission bits this test relies on") + } + + t.Run("read-only root", func(t *testing.T) { + dir := filepath.Join(t.TempDir(), "uploads") + require.NoError(t, os.Mkdir(dir, 0o500)) + t.Cleanup(func() { _ = os.Chmod(dir, 0o700) }) + + _, err := NewUploadStore(&Options{ + CorrelationIdLength: settings.CorrelationIdLengthDefault, + UploadDirectory: dir, UploadTTL: time.Hour, + }) + require.Error(t, err, "an unwritable upload directory must not start the server") + require.Contains(t, err.Error(), dir) + }) + + t.Run("sessions directory exists but is read-only", func(t *testing.T) { + // The case MkdirAll cannot catch: the directory it would have created is + // already there, so it returns nil and only a write reveals the problem. + dir := t.TempDir() + sessions := filepath.Join(dir, uploadsDirName) + require.NoError(t, os.Mkdir(sessions, 0o500)) + t.Cleanup(func() { _ = os.Chmod(sessions, 0o700) }) + + _, err := NewUploadStore(&Options{ + CorrelationIdLength: settings.CorrelationIdLengthDefault, + UploadDirectory: dir, UploadTTL: time.Hour, + }) + require.Error(t, err) + require.Contains(t, err.Error(), "not writable") + }) + + t.Run("writable root is accepted and left clean", func(t *testing.T) { + dir := t.TempDir() + store, err := NewUploadStore(&Options{ + CorrelationIdLength: settings.CorrelationIdLengthDefault, + UploadDirectory: dir, UploadTTL: time.Hour, + }) + require.NoError(t, err) + defer store.Close() + + entries, err := os.ReadDir(filepath.Join(dir, uploadsDirName)) + require.NoError(t, err) + require.Empty(t, entries, "the probe file must not be left behind") + }) +} From 9a1d7c518c29d4ddc203ec3a0c31e72051548c79 Mon Sep 17 00:00:00 2001 From: nisay759 Date: Fri, 31 Jul 2026 00:04:30 +0200 Subject: [PATCH 04/20] server: wire upload options, flags and lifecycle Adds the Upload flag group to interactsh-server and plumbs it through CLIServerOptions into server.Options. -upload forces authentication, joining -responder/-smb/-ftp/-ldap in the existing condition. Anonymous file hosting on a wildcard-TLS domain is not something to leave open by default, and the flag help says self-hosted only. The upload store is constructed before storage.New so it can install the OnEviction hook, and before the HTTP and FTP servers since both serve from its root. When no -ftp-dir is given the FTP root is pointed at the upload root, which is what makes -ftp serve uploaded files with no extra config; when the operator has pinned both to different directories we warn rather than silently serving over HTTP only. On shutdown the storage is closed first: goburrow's cache.Close blocks until every removal callback has run, so all session deletions are queued by the time the upload store drains them. Co-Authored-By: Claude The FTP root and the upload root have to be the same directory for hosted files to be reachable over ftp://, since the FTP file driver serves a real directory tree. They are compared as resolved directories rather than as the strings the operator typed: filepath.Abs and EvalSymlinks on both sides, falling back to the cleaned absolute form when a path does not exist yet, because -ftp-dir legitimately may not. Comparing the raw flag against an already absolutised root would report a mismatch for "-ud ./shared -ftp-dir ./shared", for a trailing slash, for a /./ segment and for a symlink -- four spellings of one directory -- and warning about a configuration that demonstrably works teaches the operator to ignore the warning that matters. The answer is recorded in Options.FTPServesUploads rather than recomputed, so that capability advertisement can be driven from it rather than from the -ftp flag alone. A genuine mismatch is reported with gologger.Error, not Warning: gologger orders LevelWarning above LevelInfo and filters on level <= maxLevel, so a warning is invisible unless -debug is passed, and this condition silently disables a capability the server would otherwise advertise. --- cmd/interactsh-server/main.go | 106 ++++++++++++++++++++++++++++- cmd/interactsh-server/main_test.go | 66 ++++++++++++++++++ pkg/options/server_options.go | 13 ++++ pkg/server/server.go | 6 ++ 4 files changed, 190 insertions(+), 1 deletion(-) create mode 100644 cmd/interactsh-server/main_test.go diff --git a/cmd/interactsh-server/main.go b/cmd/interactsh-server/main.go index 9e566732..31dce010 100644 --- a/cmd/interactsh-server/main.go +++ b/cmd/interactsh-server/main.go @@ -19,6 +19,7 @@ import ( _ "net/http/pprof" + units "github.com/docker/go-units" "github.com/projectdiscovery/goflags" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger/levels" @@ -104,6 +105,15 @@ func main() { flagSet.StringVar(&cliOptions.FTPDirectory, "ftp-dir", "", "ftp directory - temporary if not specified"), ) + flagSet.CreateGroup("upload", "Upload", + flagSet.BoolVar(&cliOptions.Upload, "upload", false, "enable client file upload and hosting - self-hosted servers only (authenticated)"), + flagSet.StringVarP(&cliOptions.UploadDirectory, "upload-directory", "ud", "", "directory to store uploaded files - temporary if not specified"), + flagSet.SizeVarP(&cliOptions.UploadMaxFileSize, "upload-max-file-size", "umfs", "1mb", "maximum size of a single uploaded file"), + flagSet.IntVarP(&cliOptions.UploadMaxFiles, "upload-max-files", "umf", 5, "maximum number of uploaded files per session"), + flagSet.SizeVarP(&cliOptions.UploadMaxTotalSize, "upload-max-total-size", "umts", "1gb", "maximum total size of all uploaded files on the server"), + flagSet.DurationVarP(&cliOptions.UploadTTL, "upload-ttl", "ut", 24*time.Hour, "maximum lifetime of uploaded files"), + ) + flagSet.CreateGroup("debug", "Debug", flagSet.BoolVar(&cliOptions.Version, "version", false, "show version of the project"), flagSet.BoolVar(&cliOptions.Debug, "debug", false, "start interactsh server in debug mode"), @@ -209,7 +219,7 @@ func main() { } // Requires auth if token is specified or enables it automatically for responder and smb options - if serverOptions.Token != "" || cliOptions.Responder || cliOptions.Smb || cliOptions.Ftp || cliOptions.LdapWithFullLogger { + if serverOptions.Token != "" || cliOptions.Responder || cliOptions.Smb || cliOptions.Ftp || cliOptions.LdapWithFullLogger || cliOptions.Upload { serverOptions.Auth = true } @@ -269,6 +279,54 @@ func main() { atomic.AddInt64(&serverOptions.Stats.Sessions, -1) } + // The upload store must exist before the HTTP and FTP servers are built, + // since both serve from its root. + var uploadStore *server.UploadStore + if cliOptions.Upload { + // Hosted bytes live on this instance's local filesystem and the + // capacity quota is an in-process counter, so file hosting cannot be + // combined with a storage backend shared between instances: peers + // would advertise files they do not have. + if cliOptions.RedisURL != "" { + gologger.Fatal().Msgf("-upload cannot be used with -redis-url: hosted files are stored on a single instance's local filesystem\n") + } + var err error + if uploadStore, err = server.NewUploadStore(serverOptions); err != nil { + gologger.Fatal().Msgf("could not create upload store: %s\n", err) + } + serverOptions.UploadStore = uploadStore + + // Sharing the root is what lets the existing FTP file driver serve + // uploads. If the operator pinned both to different places, say so and + // stop advertising FTP, rather than printing ftp:// payload URLs that + // resolve to nothing. + switch { + case serverOptions.FTPDirectory == "": + serverOptions.FTPDirectory = uploadStore.Root() + serverOptions.FTPServesUploads = true + default: + // Compared as resolved paths, not as the operator typed them: + // "./uploads", "/abs/uploads/" and a symlink to the same place are + // one directory, and warning about a working configuration teaches + // the operator to ignore the warning that matters. + shared, err := sameDirectory(serverOptions.FTPDirectory, uploadStore.Root()) + if err != nil { + gologger.Fatal().Msgf("could not compare ftp and upload directories: %s\n", err) + } + serverOptions.FTPServesUploads = shared + if !shared && cliOptions.Ftp { + gologger.Error().Msgf("ftp directory %s is not the upload directory %s, so uploaded files will not be served over FTP; ftp:// URLs will not be offered to clients\n", + serverOptions.FTPDirectory, uploadStore.Root()) + } + } + + // Deleting a session's files is driven by the correlation-id leaving + // the cache, whatever the reason. + storeOptions.OnEviction = func(correlationID string, _ *storage.CorrelationData) { + uploadStore.RemoveSession(correlationID) + } + } + var err error switch { case cliOptions.RedisURL != "": @@ -294,6 +352,12 @@ func main() { serverOptions.Storage = store + if uploadStore != nil { + uploadStore.Start() + gologger.Info().Msgf("Uploads enabled, hosting from %s (max %d files of %s each)\n", + uploadStore.Root(), cliOptions.UploadMaxFiles, units.BytesSize(float64(cliOptions.UploadMaxFileSize))) + } + if serverOptions.Auth { _ = serverOptions.Storage.SetID(serverOptions.Token) } @@ -503,9 +567,16 @@ func main() { c := make(chan os.Signal, 1) signal.Notify(c, os.Interrupt) for range c { + // Closed first: cache.Close blocks until every removal callback has run, + // so all session deletions are queued by the time we drain them below. if err := store.Close(); err != nil { gologger.Warning().Msgf("Couldn't close the storage: %s\n", err) } + if uploadStore != nil { + if err := uploadStore.Close(); err != nil { + gologger.Warning().Msgf("Couldn't close the upload store: %s\n", err) + } + } if pprofServer != nil { if err := pprofServer.Close(); err != nil { gologger.Warning().Msgf("Couldn't close the pprof server: %s\n", err) @@ -535,3 +606,36 @@ func getPublicIP() (string, error) { return externalIP, errors.New("couldn't find an interface configured with external ip") } + +// sameDirectory reports whether two paths name the same directory. Both are made +// absolute and symlink-resolved first, so that the FTP root and the upload root +// are compared as directories rather than as the strings the operator typed. +// +// A path that does not exist yet is compared in its cleaned absolute form: +// EvalSymlinks fails on a missing path, and that is not an error worth refusing +// to start over. +func sameDirectory(a, b string) (bool, error) { + resolve := func(p string) (string, error) { + abs, err := filepath.Abs(p) + if err != nil { + return "", err + } + resolved, err := filepath.EvalSymlinks(abs) + if err != nil { + if os.IsNotExist(err) { + return filepath.Clean(abs), nil + } + return "", err + } + return resolved, nil + } + ra, err := resolve(a) + if err != nil { + return false, err + } + rb, err := resolve(b) + if err != nil { + return false, err + } + return ra == rb, nil +} diff --git a/cmd/interactsh-server/main_test.go b/cmd/interactsh-server/main_test.go new file mode 100644 index 00000000..3746405d --- /dev/null +++ b/cmd/interactsh-server/main_test.go @@ -0,0 +1,66 @@ +package main + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +// The FTP and upload roots decide whether hosted files are reachable over ftp://, +// so they have to be compared as directories rather than as the strings the +// operator typed: warning about a working configuration teaches them to ignore +// the warning that matters. +func TestSameDirectory(t *testing.T) { + base := t.TempDir() + shared := filepath.Join(base, "shared") + require.NoError(t, os.Mkdir(shared, 0o700)) + other := filepath.Join(base, "other") + require.NoError(t, os.Mkdir(other, 0o700)) + + link := filepath.Join(base, "link") + require.NoError(t, os.Symlink(shared, link)) + + wd, err := os.Getwd() + require.NoError(t, err) + t.Cleanup(func() { _ = os.Chdir(wd) }) + require.NoError(t, os.Chdir(base)) + + t.Run("same directory in different spellings", func(t *testing.T) { + for _, pair := range [][2]string{ + {shared, shared}, + {shared, shared + string(filepath.Separator)}, + {shared, filepath.Join(base, ".", "shared")}, + {shared, filepath.Join(base, "other", "..", "shared")}, + {"./shared", "shared"}, + {"./shared", shared}, + {link, shared}, + } { + got, err := sameDirectory(pair[0], pair[1]) + require.NoError(t, err) + require.True(t, got, "%q and %q are the same directory", pair[0], pair[1]) + } + }) + + t.Run("genuinely different directories", func(t *testing.T) { + for _, pair := range [][2]string{ + {shared, other}, + {"./shared", "./other"}, + {shared, filepath.Join(base, "absent")}, + } { + got, err := sameDirectory(pair[0], pair[1]) + require.NoError(t, err) + require.False(t, got, "%q and %q are different directories", pair[0], pair[1]) + } + }) + + // EvalSymlinks fails on a path that does not exist, which is not a reason to + // refuse to start: -ftp-dir may legitimately not exist yet. + t.Run("missing paths compare by cleaned absolute form", func(t *testing.T) { + missing := filepath.Join(base, "not-created-yet") + got, err := sameDirectory(missing, missing+string(filepath.Separator)) + require.NoError(t, err) + require.True(t, got) + }) +} diff --git a/pkg/options/server_options.go b/pkg/options/server_options.go index 96352cb9..f8aa1031 100644 --- a/pkg/options/server_options.go +++ b/pkg/options/server_options.go @@ -2,6 +2,7 @@ package options import ( "net" + "time" "github.com/projectdiscovery/goflags" "github.com/projectdiscovery/gologger" @@ -64,6 +65,12 @@ type CLIServerOptions struct { NoVersionHeader bool HeaderServer string DefaultHTTPResponseFile string + Upload bool + UploadDirectory string + UploadMaxFileSize goflags.Size + UploadMaxFiles int + UploadMaxTotalSize goflags.Size + UploadTTL time.Duration } func (cliServerOptions *CLIServerOptions) AsServerOptions() *server.Options { @@ -119,6 +126,12 @@ func (cliServerOptions *CLIServerOptions) AsServerOptions() *server.Options { NoVersionHeader: cliServerOptions.NoVersionHeader, HeaderServer: cliServerOptions.HeaderServer, DefaultHTTPResponseFile: cliServerOptions.DefaultHTTPResponseFile, + Upload: cliServerOptions.Upload, + UploadDirectory: cliServerOptions.UploadDirectory, + UploadMaxFileSize: int64(cliServerOptions.UploadMaxFileSize), + UploadMaxFiles: cliServerOptions.UploadMaxFiles, + UploadMaxTotalSize: int64(cliServerOptions.UploadMaxTotalSize), + UploadTTL: cliServerOptions.UploadTTL, } } diff --git a/pkg/server/server.go b/pkg/server/server.go index 83981776..df1708d2 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -125,6 +125,12 @@ type Options struct { UploadTTL time.Duration // UploadStore serves and stores uploaded files. Nil when uploads are disabled. UploadStore *UploadStore + // FTPServesUploads reports whether the FTP root and the upload root resolve + // to the same directory, which is what makes hosted files reachable over + // ftp://. Derived at startup from the resolved paths, never from the flags + // as written, so that two spellings of one directory are not mistaken for + // two directories. + FTPServesUploads bool ACMEStore *acme.Provider Stats *Metrics From 3b79661ab3ceddfcbacc976a9beecdccf51d4ad1 Mon Sep 17 00:00:00 2001 From: nisay759 Date: Fri, 31 Jul 2026 00:07:26 +0200 Subject: [PATCH 05/20] server: add /upload endpoint and capability advertisement POST /upload accepts JSON with base64 file bodies, authenticated by the correlation-id and secret-key pair -- the same ownership proof RemoveID requires, so only the client that owns a session can attach files to it. The route is registered on the mux rather than under "/", so it never passes through the logger middleware; that middleware dumps whole requests into interaction records, which for an upload endpoint would mean storing and re-encrypting every uploaded file. It is registered even when uploads are disabled, so that a request to a server without -upload is answered with 501 instead of falling through to exactly that path. Clients use the 501 as the signal to stop rather than blindly posting files at a server that will quietly swallow them. JSON with base64 rather than multipart: it matches every other endpoint, needs no new dependency, and at five 1MB files the 33% overhead is immaterial. Everything is decoded and validated before storage is touched, so a bad file part way through a batch cannot leave a session half-populated. The writes themselves run inside UpdateUploads, under the correlation-id lock, so the per-session quota check and the commit are atomic against a concurrent upload for the same session. Re-uploading a name replaces it and reuses its slot rather than consuming another. Status codes are distinct enough for the client to act on: 501 disabled, 404 unknown session, 403 wrong secret, 413 over a size or count limit, 507 server capacity exhausted, 400 for anything malformed. Registration now answers with RegisterResponse carrying a Capabilities block, so a client learns whether uploads are available, and the limits, at registration rather than by trial and error. The message field keeps its exact previous value, which older clients match on, and an older server simply yields a nil Capabilities. Co-Authored-By: Claude A request to /upload that no legitimate client could have sent is a target poking at the endpoint, so it is recorded as an interaction rather than being swallowed by the 401 or the 501. Nothing legitimate arrives there to confuse it with: the client reads the advertised capabilities and refuses to send when uploads are off, and -upload forces -auth with a randomly generated token, so a target cannot authenticate. The request body is summarised as its length rather than stored -- it is attacker-controlled and may be megabytes, which is the whole reason this route stays off the logger middleware. A 256KB probe retains ~240 bytes. Authenticated uploads are deliberately not recorded. They are the operator's own traffic, and filing them as interactions would attribute the operator's actions to the target, which is a false positive in the evidence rather than just noise. That is why the token check sits in uploadHandler rather than authMiddleware: the middleware writes its 401 and returns, leaving nowhere to record from. It also keeps the check reachable from tests, which call uploadHandler directly and never assemble the middleware chain. extractCorrelationID moves here from the file-serving commit, since this is now the first caller: recordUploadProbe has to resolve a session before it can file anything, and drops the interaction when the host carries no correlation id, because handleInteraction slices one out of uniqueID unconditionally. The FTP capability is advertised from Options.FTPServesUploads rather than from the -ftp flag, so it answers the question the client is actually asking: whether an ftp:// URL for a hosted file is worth printing. Taking it from -ftp alone would advertise FTP on a server whose FTP root is a different directory from the upload root, and the client would print a payload URL that resolves to nothing -- the target follows it, gets a 550, and the operator reads the silence as "not vulnerable", which is indistinguishable from a target that is not vulnerable. --- pkg/options/server_options.go | 1 + pkg/server/http_server.go | 26 ++- pkg/server/server.go | 12 ++ pkg/server/upload_handler.go | 321 ++++++++++++++++++++++++++++++ pkg/server/upload_handler_test.go | 292 +++++++++++++++++++++++++++ pkg/server/upload_probe_test.go | 122 ++++++++++++ pkg/server/util.go | 30 +++ 7 files changed, 803 insertions(+), 1 deletion(-) create mode 100644 pkg/server/upload_handler.go create mode 100644 pkg/server/upload_handler_test.go create mode 100644 pkg/server/upload_probe_test.go diff --git a/pkg/options/server_options.go b/pkg/options/server_options.go index f8aa1031..110b9f18 100644 --- a/pkg/options/server_options.go +++ b/pkg/options/server_options.go @@ -126,6 +126,7 @@ func (cliServerOptions *CLIServerOptions) AsServerOptions() *server.Options { NoVersionHeader: cliServerOptions.NoVersionHeader, HeaderServer: cliServerOptions.HeaderServer, DefaultHTTPResponseFile: cliServerOptions.DefaultHTTPResponseFile, + Ftp: cliServerOptions.Ftp, Upload: cliServerOptions.Upload, UploadDirectory: cliServerOptions.UploadDirectory, UploadMaxFileSize: int64(cliServerOptions.UploadMaxFileSize), diff --git a/pkg/server/http_server.go b/pkg/server/http_server.go index 832278aa..bc77744d 100644 --- a/pkg/server/http_server.go +++ b/pkg/server/http_server.go @@ -84,6 +84,20 @@ func NewHTTPServer(options *Options) (*HTTPServer, error) { router.Handle("/register", server.corsMiddleware(server.authMiddleware(http.HandlerFunc(server.registerHandler)))) router.Handle("/deregister", server.corsMiddleware(server.authMiddleware(http.HandlerFunc(server.deregisterHandler)))) router.Handle("/poll", server.corsMiddleware(server.authMiddleware(http.HandlerFunc(server.pollHandler)))) + // Registered even when uploads are disabled, so that an upload request to a + // server without -upload gets a clean 501 rather than falling through to + // "/", where the logger middleware would persist the whole file body as an + // interaction record. + // + // Unlike the other authenticated routes this one is not wrapped in + // authMiddleware: it checks the token itself, because a request that fails + // the check has to be recorded as an interaction before the 401 and the + // middleware returns too early to allow that. -upload forces -auth with a + // random token, and the client refuses to send an upload to a server whose + // advertised capabilities say uploads are off, so anything unauthenticated + // arriving here is a target probing the endpoint -- exactly what we exist to + // record. + router.Handle("/upload", server.corsMiddleware(http.HandlerFunc(server.uploadHandler))) if server.options.EnableMetrics { router.Handle("/metrics", server.corsMiddleware(server.authMiddleware(http.HandlerFunc(server.metricsHandler)))) } @@ -399,7 +413,17 @@ func (h *HTTPServer) registerHandler(w http.ResponseWriter, req *http.Request) { } atomic.AddInt64(&h.options.Stats.Sessions, 1) atomic.AddInt64(&h.options.Stats.SessionsTotal, 1) - jsonMsg(w, "registration successful", http.StatusOK) + + // Capabilities ride along on the registration response so the client knows + // whether uploads are available without a second round trip. Older clients + // read only "message" and ignore the extra key. + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.Header().Set("X-Content-Type-Options", "nosniff") + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(&RegisterResponse{ + Message: "registration successful", + Capabilities: h.capabilities(), + }) gologger.Debug().Msgf("Registered correlationID %s for key\n", r.CorrelationID) } diff --git a/pkg/server/server.go b/pkg/server/server.go index df1708d2..773ad9d8 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -111,6 +111,9 @@ type Options struct { // DefaultHTTPResponseFile is a file to serve for all HTTP requests (takes priority over other options) DefaultHTTPResponseFile string + // Ftp indicates the FTP server is enabled, so uploaded files are also + // reachable over ftp:// + Ftp bool // Upload enables the client file upload endpoint and file hosting Upload bool // UploadDirectory is the root directory uploaded files are stored under @@ -141,6 +144,15 @@ type Options struct { } type OnResultCallback func(out interface{}) +// UploadStorage returns the configured storage backend's upload-tracking +// capability, or nil when the backend does not implement it. Only +// instance-local backends do, since hosted bytes live on the local filesystem; +// see storage.UploadStorage. +func (options *Options) UploadStorage() storage.UploadStorage { + uploadStorage, _ := options.Storage.(storage.UploadStorage) + return uploadStorage +} + func (options *Options) GetIdLength() int { return options.CorrelationIdLength + options.CorrelationIdNonceLength } diff --git a/pkg/server/upload_handler.go b/pkg/server/upload_handler.go new file mode 100644 index 00000000..0429ec0e --- /dev/null +++ b/pkg/server/upload_handler.go @@ -0,0 +1,321 @@ +package server + +import ( + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "net" + "net/http" + "net/http/httputil" + "path" + "strings" + "sync/atomic" + "time" + + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/interactsh/pkg/storage" +) + +// Capabilities advertises optional server features to clients at registration. +// Clients use it to decide whether to attempt an upload at all, and which URLs +// are worth printing. +type Capabilities struct { + Upload bool `json:"upload"` + UploadMaxFileSize int64 `json:"upload-max-file-size,omitempty"` + UploadMaxFiles int `json:"upload-max-files,omitempty"` + FTP bool `json:"ftp"` +} + +// RegisterResponse is the response to a client registration. The bare +// {"message": ...} shape older clients expect is preserved, so an old client +// against a new server simply ignores the extra key, and a new client against +// an old server sees a nil Capabilities. +type RegisterResponse struct { + Message string `json:"message"` + Capabilities *Capabilities `json:"capabilities,omitempty"` +} + +// UploadFileRequest is a single file within an upload request. +type UploadFileRequest struct { + Name string `json:"name"` + Data string `json:"data"` // base64 +} + +// UploadRequest is a request to host files against a correlation ID. +type UploadRequest struct { + CorrelationID string `json:"correlation-id"` + SecretKey string `json:"secret-key"` + Files []UploadFileRequest `json:"files"` +} + +// UploadedFileResponse describes one hosted file. Paths rather than URLs: the +// server does not know which of its domains, or which scheme, the client will +// use to reference the file. +type UploadedFileResponse struct { + Name string `json:"name"` + Size int64 `json:"size"` + SHA256 string `json:"sha256"` + HTTPPath string `json:"http-path"` + FTPPath string `json:"ftp-path"` +} + +// UploadResponse is the response to a successful upload. +type UploadResponse struct { + Message string `json:"message"` + Files []UploadedFileResponse `json:"files"` +} + +// capabilities describes what this server offers. +func (h *HTTPServer) capabilities() *Capabilities { + // FTP is advertised only when it can actually reach the hosted files. The + // client uses this to decide whether an ftp:// URL is worth printing, and a + // URL pointing into a directory the FTP server does not serve is worse than + // no URL at all: the target follows it, gets a 550, and the operator reads + // the silence as "not vulnerable". + c := &Capabilities{FTP: h.options.Ftp && h.options.FTPServesUploads} + // Advertised only when the storage backend can track uploads too, so the + // server never announces a capability uploadHandler would refuse with 501. + if store := h.options.UploadStore; store != nil && h.options.UploadStorage() != nil { + c.Upload = true + c.UploadMaxFileSize = store.MaxFileSize() + c.UploadMaxFiles = store.MaxFiles() + } + return c +} + +// maxUploadRequestBytes bounds the request body: every file at its size limit, +// base64 expanded, plus room for JSON framing. +func maxUploadRequestBytes(store *UploadStore) int64 { + return int64(store.MaxFiles())*store.MaxFileSize()*4/3 + 8192 +} + +// uploadHandler stores files against a correlation ID for later hosting. +// +// Registered as its own route rather than under "/" so that it never passes +// through the logger middleware, which would otherwise dump the entire request +// body -- i.e. every uploaded file -- into an interaction record. It is +// registered even when uploads are disabled, precisely so that a request to a +// non-upload server gets a clean 501 instead of falling through to that path. +func (h *HTTPServer) uploadHandler(w http.ResponseWriter, req *http.Request) { + // The token check lives here rather than in authMiddleware so that a request + // failing it can be recorded first; see the route registration for why that + // matters. Recording is deliberately limited to requests no legitimate client + // could have sent: an authenticated upload is the operator's own traffic, and + // filing it as an interaction would attribute their action to the target. + if !h.checkToken(req) { + h.recordUploadProbe(req, http.StatusUnauthorized, "") + w.WriteHeader(http.StatusUnauthorized) + return + } + + store := h.options.UploadStore + uploadStorage := h.options.UploadStorage() + // uploadStorage is nil when the configured storage backend cannot track + // uploads (a shared backend such as Redis). Treated the same as uploads + // being switched off, so the client sees the usual capability signal. + if store == nil || uploadStorage == nil { + // A client that has seen this server's capabilities does not send an + // upload here at all, so this is a probe too. + const message = "file upload is not enabled on this server" + h.recordUploadProbe(req, http.StatusNotImplemented, message) + jsonError(w, message, http.StatusNotImplemented) + return + } + if req.Method != http.MethodPost { + jsonError(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + + req.Body = http.MaxBytesReader(w, req.Body, maxUploadRequestBytes(store)) + + r := &UploadRequest{} + if err := json.NewDecoder(req.Body).Decode(r); err != nil { + var maxErr *http.MaxBytesError + if errors.As(err, &maxErr) { + jsonError(w, "upload request too large", http.StatusRequestEntityTooLarge) + return + } + jsonError(w, fmt.Sprintf("could not decode json body: %s", err), http.StatusBadRequest) + return + } + if len(r.Files) == 0 { + jsonError(w, "no files provided", http.StatusBadRequest) + return + } + if len(r.Files) > store.MaxFiles() { + jsonError(w, fmt.Sprintf("too many files, limit is %d", store.MaxFiles()), http.StatusRequestEntityTooLarge) + return + } + + // Decode and validate everything before touching storage, so a malformed + // request cannot leave a session half-populated. + decoded := make([][]byte, len(r.Files)) + seen := make(map[string]struct{}, len(r.Files)) + for i, f := range r.Files { + if !isSafeUploadName(f.Name) { + jsonError(w, fmt.Sprintf("invalid file name %q", f.Name), http.StatusBadRequest) + return + } + if _, dup := seen[f.Name]; dup { + jsonError(w, fmt.Sprintf("duplicate file name %q", f.Name), http.StatusBadRequest) + return + } + seen[f.Name] = struct{}{} + + data, err := base64.StdEncoding.DecodeString(f.Data) + if err != nil { + jsonError(w, fmt.Sprintf("could not decode data for %q: %s", f.Name, err), http.StatusBadRequest) + return + } + if len(data) == 0 { + jsonError(w, fmt.Sprintf("file %q is empty", f.Name), http.StatusBadRequest) + return + } + if int64(len(data)) > store.MaxFileSize() { + jsonError(w, fmt.Sprintf("file %q exceeds the %d byte limit", f.Name, store.MaxFileSize()), http.StatusRequestEntityTooLarge) + return + } + decoded[i] = data + } + + var ( + response []UploadedFileResponse + ftpPrefix = path.Join("/", uploadsDirName, r.CorrelationID) + ) + + // The disk writes happen inside UpdateUploads, i.e. under the correlation + // ID's lock, so the per-session quota check and the commit are atomic with + // respect to a concurrent upload for the same session. + err := uploadStorage.UpdateUploads(r.CorrelationID, r.SecretKey, func(existing []storage.UploadedFile) ([]storage.UploadedFile, error) { + updated := append([]storage.UploadedFile(nil), existing...) + + for i, f := range r.Files { + var existingSize int64 + idx := -1 + for j, u := range updated { + if u.Name == f.Name { + existingSize, idx = u.Size, j + break + } + } + // Replacing a name reuses its slot rather than consuming a new one. + if idx == -1 && len(updated) >= store.MaxFiles() { + return nil, uploadErr(UploadErrTooManyFiles, + "session already holds %d files, limit is %d", len(updated), store.MaxFiles()) + } + + size, sum, err := store.Save(r.CorrelationID, f.Name, decoded[i], existingSize) + if err != nil { + return nil, err + } + + record := storage.UploadedFile{Name: f.Name, Size: size, SHA256: sum, Timestamp: time.Now()} + if idx == -1 { + updated = append(updated, record) + } else { + updated[idx] = record + } + response = append(response, UploadedFileResponse{ + Name: f.Name, + Size: size, + SHA256: sum, + HTTPPath: path.Join("/f", f.Name), + FTPPath: path.Join(ftpPrefix, f.Name), + }) + } + return updated, nil + }) + + if err != nil { + h.writeUploadError(w, err) + return + } + + gologger.Debug().Msgf("Stored %d uploaded file(s) for %s\n", len(response), r.CorrelationID) + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.Header().Set("X-Content-Type-Options", "nosniff") + _ = json.NewEncoder(w).Encode(&UploadResponse{Message: "upload successful", Files: response}) +} + +// recordUploadProbe records an interaction for a request to /upload that no +// legitimate client could have sent, so that a target poking at the endpoint +// reaches the operator's stream instead of vanishing behind a 401 or a 501. +// +// The request body is summarised rather than stored. It is attacker-controlled +// and may be megabytes -- persisting it is precisely what keeping /upload off the +// logger middleware avoids -- but its size is evidence worth keeping. +// +// status and body describe the reply the caller is about to send, so the stored +// record cannot drift from what the target actually received. +func (h *HTTPServer) recordUploadProbe(req *http.Request, status int, body string) { + uniqueID, fullID := h.options.extractCorrelationID(req.Host) + if uniqueID == "" { + // Nothing to attribute it to: interactions are indexed by correlation id, + // and handleInteraction slices one out of uniqueID unconditionally. + return + } + atomic.AddUint64(&h.options.Stats.Http, 1) + + var host string + if originIP := req.Header.Get(h.options.OriginIPHeader); originIP != "" { + host = originIP + } else { + host, _, _ = net.SplitHostPort(req.RemoteAddr) + } + + reqDump, _ := httputil.DumpRequest(req, false) + reqString := string(reqDump) + if req.ContentLength > 0 { + reqString += fmt.Sprintf("[request body elided: %d bytes]\n", req.ContentLength) + } + + var resp strings.Builder + fmt.Fprintf(&resp, "HTTP/1.1 %d %s\r\n", status, http.StatusText(status)) + if body != "" { + // Mirrors jsonBody, which is what the caller writes. + encoded, err := json.Marshal(map[string]interface{}{"error": body}) + if err != nil { + return + } + fmt.Fprintf(&resp, "Content-Type: application/json; charset=utf-8\r\n") + fmt.Fprintf(&resp, "X-Content-Type-Options: nosniff\r\n") + fmt.Fprintf(&resp, "Content-Length: %d\r\n\r\n%s\n", len(encoded)+1, encoded) + } else { + resp.WriteString("\r\n") + } + + h.handleInteraction(req, uniqueID, fullID, reqString, resp.String(), host) +} + +// writeUploadError maps a failure to a status code the client can act on. +func (h *HTTPServer) writeUploadError(w http.ResponseWriter, err error) { + switch { + case errors.Is(err, storage.ErrCorrelationIdNotFound): + jsonError(w, "unknown correlation-id", http.StatusNotFound) + return + case errors.Is(err, storage.ErrInvalidSecretKey): + // Distinct from 400 so the client can tell "wrong session" apart from + // "malformed request". + jsonError(w, "invalid secret key for correlation-id", http.StatusForbidden) + return + } + + var ue *UploadError + if errors.As(err, &ue) { + switch ue.Kind { + case UploadErrBadName: + jsonError(w, ue.Error(), http.StatusBadRequest) + case UploadErrTooLarge, UploadErrTooManyFiles: + jsonError(w, ue.Error(), http.StatusRequestEntityTooLarge) + case UploadErrOutOfSpace: + jsonError(w, ue.Error(), http.StatusInsufficientStorage) + default: + jsonError(w, ue.Error(), http.StatusBadRequest) + } + return + } + + gologger.Warning().Msgf("Could not store uploaded files: %s\n", err) + jsonError(w, "could not store uploaded files", http.StatusInternalServerError) +} diff --git a/pkg/server/upload_handler_test.go b/pkg/server/upload_handler_test.go new file mode 100644 index 00000000..98a36547 --- /dev/null +++ b/pkg/server/upload_handler_test.go @@ -0,0 +1,292 @@ +package server + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/base64" + "encoding/json" + "encoding/pem" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/google/uuid" + "github.com/projectdiscovery/interactsh/pkg/settings" + "github.com/projectdiscovery/interactsh/pkg/storage" + "github.com/rs/xid" + "github.com/stretchr/testify/require" +) + +// testPublicKey returns a base64-encoded PEM RSA public key accepted by +// Storage.SetIDPublicKey. +func testPublicKey(t *testing.T) string { + t.Helper() + + priv, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err, "could not generate rsa key") + + pubkeyBytes, err := x509.MarshalPKIXPublicKey(priv.Public()) + require.NoError(t, err, "could not marshal public key") + + pubkeyPem := pem.EncodeToMemory(&pem.Block{Type: "RSA PUBLIC KEY", Bytes: pubkeyBytes}) + return base64.StdEncoding.EncodeToString(pubkeyPem) +} + +// uploadTestServer returns an HTTPServer with uploads enabled and a registered +// session, plus that session's correlation ID and secret. +func uploadTestServer(t *testing.T, enableUploads bool) (*HTTPServer, string, string) { + t.Helper() + + store, err := storage.New(&storage.Options{EvictionTTL: time.Hour}) + require.NoError(t, err) + t.Cleanup(func() { _ = store.Close() }) + + options := &Options{ + Storage: store, + Stats: &Metrics{}, + CorrelationIdLength: settings.CorrelationIdLengthDefault, + CorrelationIdNonceLength: settings.CorrelationIdNonceLengthDefault, + UploadDirectory: t.TempDir(), + UploadMaxFileSize: 1024, + UploadMaxFiles: 3, + UploadMaxTotalSize: 1 << 20, + UploadTTL: time.Hour, + } + if enableUploads { + us, err := NewUploadStore(options) + require.NoError(t, err) + t.Cleanup(func() { _ = us.Close() }) + options.Upload = true + options.UploadStore = us + } + + h := &HTTPServer{options: options} + + secret := uuid.New().String() + correlationID := xid.New().String() + require.NoError(t, store.SetIDPublicKey(correlationID, secret, testPublicKey(t))) + + return h, correlationID, secret +} + +func uploadBody(t *testing.T, correlationID, secret string, files map[string][]byte) string { + t.Helper() + + req := UploadRequest{CorrelationID: correlationID, SecretKey: secret} + for name, data := range files { + req.Files = append(req.Files, UploadFileRequest{ + Name: name, + Data: base64.StdEncoding.EncodeToString(data), + }) + } + encoded, err := json.Marshal(req) + require.NoError(t, err) + return string(encoded) +} + +func doUpload(t *testing.T, h *HTTPServer, body string) *http.Response { + t.Helper() + req := httptest.NewRequest(http.MethodPost, "http://example.com/upload", strings.NewReader(body)) + w := httptest.NewRecorder() + h.uploadHandler(w, req) + return w.Result() +} + +func TestUploadHandler(t *testing.T) { + t.Run("stores a file", func(t *testing.T) { + h, id, secret := uploadTestServer(t, true) + content := []byte("") + + resp := doUpload(t, h, uploadBody(t, id, secret, map[string][]byte{"evil.dtd": content})) + require.Equal(t, http.StatusOK, resp.StatusCode) + + out := &UploadResponse{} + require.NoError(t, json.NewDecoder(resp.Body).Decode(out)) + require.Len(t, out.Files, 1) + require.Equal(t, "evil.dtd", out.Files[0].Name) + require.EqualValues(t, len(content), out.Files[0].Size) + require.Equal(t, "/f/evil.dtd", out.Files[0].HTTPPath) + require.Equal(t, "/"+uploadsDirName+"/"+id+"/evil.dtd", out.Files[0].FTPPath) + + files, ok := h.options.UploadStorage().ListUploads(id) + require.True(t, ok) + require.Len(t, files, 1, "metadata should be recorded against the session") + }) + + t.Run("501 when uploads disabled", func(t *testing.T) { + h, id, secret := uploadTestServer(t, false) + resp := doUpload(t, h, uploadBody(t, id, secret, map[string][]byte{"a.dtd": []byte("x")})) + require.Equal(t, http.StatusNotImplemented, resp.StatusCode, + "the client uses 501 as the capability signal") + }) + + t.Run("404 for unknown correlation id", func(t *testing.T) { + h, _, secret := uploadTestServer(t, true) + resp := doUpload(t, h, uploadBody(t, xid.New().String(), secret, map[string][]byte{"a.dtd": []byte("x")})) + require.Equal(t, http.StatusNotFound, resp.StatusCode) + }) + + t.Run("403 for wrong secret", func(t *testing.T) { + h, id, _ := uploadTestServer(t, true) + resp := doUpload(t, h, uploadBody(t, id, uuid.New().String(), map[string][]byte{"a.dtd": []byte("x")})) + require.Equal(t, http.StatusForbidden, resp.StatusCode) + }) + + t.Run("413 for oversize file", func(t *testing.T) { + h, id, secret := uploadTestServer(t, true) + resp := doUpload(t, h, uploadBody(t, id, secret, map[string][]byte{"big.bin": make([]byte, 2048)})) + require.Equal(t, http.StatusRequestEntityTooLarge, resp.StatusCode) + }) + + t.Run("413 for too many files in one request", func(t *testing.T) { + h, id, secret := uploadTestServer(t, true) + files := map[string][]byte{} + for i := 0; i < 5; i++ { + files[fmt.Sprintf("f%d.dtd", i)] = []byte("x") + } + resp := doUpload(t, h, uploadBody(t, id, secret, files)) + require.Equal(t, http.StatusRequestEntityTooLarge, resp.StatusCode) + }) + + t.Run("413 when session quota is reached across requests", func(t *testing.T) { + h, id, secret := uploadTestServer(t, true) + for i := 0; i < 3; i++ { + resp := doUpload(t, h, uploadBody(t, id, secret, + map[string][]byte{fmt.Sprintf("f%d.dtd", i): []byte("x")})) + require.Equal(t, http.StatusOK, resp.StatusCode) + } + resp := doUpload(t, h, uploadBody(t, id, secret, map[string][]byte{"overflow.dtd": []byte("x")})) + require.Equal(t, http.StatusRequestEntityTooLarge, resp.StatusCode) + }) + + t.Run("replacing a name does not consume a new slot", func(t *testing.T) { + h, id, secret := uploadTestServer(t, true) + for i := 0; i < 3; i++ { + resp := doUpload(t, h, uploadBody(t, id, secret, + map[string][]byte{fmt.Sprintf("f%d.dtd", i): []byte("x")})) + require.Equal(t, http.StatusOK, resp.StatusCode) + } + resp := doUpload(t, h, uploadBody(t, id, secret, map[string][]byte{"f0.dtd": []byte("replaced")})) + require.Equal(t, http.StatusOK, resp.StatusCode, "overwriting an existing name must be allowed at quota") + + files, _ := h.options.UploadStorage().ListUploads(id) + require.Len(t, files, 3) + }) + + t.Run("400 for traversal name", func(t *testing.T) { + h, id, secret := uploadTestServer(t, true) + for _, name := range []string{"../evil", "a/b", "/etc/passwd", `..\evil`, ""} { + resp := doUpload(t, h, uploadBody(t, id, secret, map[string][]byte{name: []byte("x")})) + require.Equal(t, http.StatusBadRequest, resp.StatusCode, "name %q must be rejected", name) + } + }) + + t.Run("400 for malformed base64", func(t *testing.T) { + h, id, secret := uploadTestServer(t, true) + body := fmt.Sprintf(`{"correlation-id":%q,"secret-key":%q,"files":[{"name":"a.dtd","data":"!!!not base64!!!"}]}`, id, secret) + resp := doUpload(t, h, body) + require.Equal(t, http.StatusBadRequest, resp.StatusCode) + }) + + t.Run("400 for duplicate names in one request", func(t *testing.T) { + h, id, secret := uploadTestServer(t, true) + body := fmt.Sprintf( + `{"correlation-id":%q,"secret-key":%q,"files":[{"name":"a.dtd","data":"eA=="},{"name":"a.dtd","data":"eQ=="}]}`, + id, secret) + resp := doUpload(t, h, body) + require.Equal(t, http.StatusBadRequest, resp.StatusCode) + }) + + t.Run("400 for empty file list", func(t *testing.T) { + h, id, secret := uploadTestServer(t, true) + resp := doUpload(t, h, uploadBody(t, id, secret, nil)) + require.Equal(t, http.StatusBadRequest, resp.StatusCode) + }) + + t.Run("405 for non-POST", func(t *testing.T) { + h, _, _ := uploadTestServer(t, true) + req := httptest.NewRequest(http.MethodGet, "http://example.com/upload", nil) + w := httptest.NewRecorder() + h.uploadHandler(w, req) + require.Equal(t, http.StatusMethodNotAllowed, w.Result().StatusCode) + }) + + // A failure partway through must not leave the session holding some files. + t.Run("rejects the whole request if any file is invalid", func(t *testing.T) { + h, id, secret := uploadTestServer(t, true) + body := fmt.Sprintf( + `{"correlation-id":%q,"secret-key":%q,"files":[{"name":"good.dtd","data":"eA=="},{"name":"../bad","data":"eQ=="}]}`, + id, secret) + resp := doUpload(t, h, body) + require.Equal(t, http.StatusBadRequest, resp.StatusCode) + + files, _ := h.options.UploadStorage().ListUploads(id) + require.Empty(t, files, "no file should have been stored") + }) +} + +func TestRegisterAdvertisesCapabilities(t *testing.T) { + t.Run("uploads enabled", func(t *testing.T) { + h, _, _ := uploadTestServer(t, true) + h.options.Ftp = true + // Set at startup once the FTP and upload roots are known to be the same + // directory; without it FTP is not advertised for hosted files. + h.options.FTPServesUploads = true + + body := fmt.Sprintf(`{"public-key":%q,"secret-key":%q,"correlation-id":%q}`, + testPublicKey(t), uuid.New().String(), xid.New().String()) + w := httptest.NewRecorder() + h.registerHandler(w, httptest.NewRequest(http.MethodPost, "http://example.com/register", strings.NewReader(body))) + + resp := w.Result() + require.Equal(t, http.StatusOK, resp.StatusCode) + + out := &RegisterResponse{} + require.NoError(t, json.NewDecoder(resp.Body).Decode(out)) + require.Equal(t, "registration successful", out.Message, + "legacy clients match on this exact string") + require.NotNil(t, out.Capabilities) + require.True(t, out.Capabilities.Upload) + require.True(t, out.Capabilities.FTP) + require.EqualValues(t, 1024, out.Capabilities.UploadMaxFileSize) + require.Equal(t, 3, out.Capabilities.UploadMaxFiles) + }) + + // An ftp:// URL that resolves to nothing is worse than no URL: the target + // follows it, gets a 550, and the operator reads the silence as "not + // vulnerable". So a split root must not be advertised. + t.Run("ftp not advertised when it cannot reach the uploads", func(t *testing.T) { + h, _, _ := uploadTestServer(t, true) + h.options.Ftp = true + h.options.FTPServesUploads = false + + body := fmt.Sprintf(`{"public-key":%q,"secret-key":%q,"correlation-id":%q}`, + testPublicKey(t), uuid.New().String(), xid.New().String()) + w := httptest.NewRecorder() + h.registerHandler(w, httptest.NewRequest(http.MethodPost, "http://example.com/register", strings.NewReader(body))) + + out := &RegisterResponse{} + require.NoError(t, json.NewDecoder(w.Result().Body).Decode(out)) + require.NotNil(t, out.Capabilities) + require.True(t, out.Capabilities.Upload, "http hosting still works") + require.False(t, out.Capabilities.FTP, "a split root must not be advertised as ftp-capable") + }) + + t.Run("uploads disabled", func(t *testing.T) { + h, _, _ := uploadTestServer(t, false) + + body := fmt.Sprintf(`{"public-key":%q,"secret-key":%q,"correlation-id":%q}`, + testPublicKey(t), uuid.New().String(), xid.New().String()) + w := httptest.NewRecorder() + h.registerHandler(w, httptest.NewRequest(http.MethodPost, "http://example.com/register", strings.NewReader(body))) + + out := &RegisterResponse{} + require.NoError(t, json.NewDecoder(w.Result().Body).Decode(out)) + require.NotNil(t, out.Capabilities) + require.False(t, out.Capabilities.Upload) + }) +} diff --git a/pkg/server/upload_probe_test.go b/pkg/server/upload_probe_test.go new file mode 100644 index 00000000..ccd146e4 --- /dev/null +++ b/pkg/server/upload_probe_test.go @@ -0,0 +1,122 @@ +package server + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// payloadHost builds a host of the shape a client payload URL would use. +func payloadHost(correlationID string) string { + return correlationID + strings.Repeat("a", 13) + ".oast.test" +} + +// probeUpload issues an upload request the way a target would: against a payload +// host, with whatever token the caller supplies. +func probeUpload(t *testing.T, h *HTTPServer, host, token, body string) *http.Response { + t.Helper() + req := httptest.NewRequest(http.MethodPost, "/upload", strings.NewReader(body)) + req.Host = host + req.RemoteAddr = "203.0.113.9:4444" + if token != "" { + req.Header.Set("Authorization", token) + } + w := httptest.NewRecorder() + h.uploadHandler(w, req) + return w.Result() +} + +// storedInteractions returns the interaction records held for a correlation id. +func storedInteractions(t *testing.T, h *HTTPServer, correlationID string) []string { + t.Helper() + item, err := h.options.Storage.GetCacheItem(correlationID) + require.NoError(t, err) + return item.Data +} + +// A legitimate client never reaches /upload unauthenticated, nor at all on a +// server whose advertised capabilities say uploads are off. So anything arriving +// there is a target probing the endpoint, and the operator should see it. +func TestUploadProbeRecording(t *testing.T) { + payload := `{"correlation-id":"probe","secret-key":"x","files":[{"name":"a.dtd","data":"QUFBQUFB"}]}` + + t.Run("unauthenticated probe is recorded and refused", func(t *testing.T) { + h, id, _ := uploadTestServer(t, true) + h.options.Auth = true + h.options.Token = "server-token" + + resp := probeUpload(t, h, payloadHost(id), "", payload) + require.Equal(t, http.StatusUnauthorized, resp.StatusCode) + + data := storedInteractions(t, h, id) + require.Len(t, data, 1, "a probe of /upload must reach the operator's stream") + + record := &Interaction{} + require.NoError(t, json.Unmarshal([]byte(data[0]), record)) + // UniqueID carries the nonce as well; the record is filed under the id. + require.True(t, strings.HasPrefix(record.UniqueID, id), "got %q", record.UniqueID) + require.Contains(t, record.RawRequest, "POST /upload") + require.Contains(t, record.RawResponse, "401 Unauthorized", + "the record must state the status the target actually received") + require.Equal(t, uint64(1), h.options.Stats.Http) + }) + + t.Run("probe against a server without uploads is recorded", func(t *testing.T) { + h, id, _ := uploadTestServer(t, false) + + resp := probeUpload(t, h, payloadHost(id), "", payload) + require.Equal(t, http.StatusNotImplemented, resp.StatusCode) + + data := storedInteractions(t, h, id) + require.Len(t, data, 1) + record := &Interaction{} + require.NoError(t, json.Unmarshal([]byte(data[0]), record)) + require.Contains(t, record.RawResponse, "501 Not Implemented") + require.Contains(t, record.RawResponse, "file upload is not enabled on this server") + }) + + t.Run("the request body is summarised, never stored", func(t *testing.T) { + h, id, _ := uploadTestServer(t, false) + big := strings.Repeat("Q", 200000) + body := `{"files":[{"name":"a.dtd","data":"` + big + `"}]}` + + resp := probeUpload(t, h, payloadHost(id), "", body) + require.Equal(t, http.StatusNotImplemented, resp.StatusCode) + + record := &Interaction{} + require.NoError(t, json.Unmarshal([]byte(storedInteractions(t, h, id)[0]), record)) + require.NotContains(t, record.RawRequest, big, + "an attacker-controlled body must not be persisted") + require.Contains(t, record.RawRequest, fmt.Sprintf("[request body elided: %d bytes]", len(body))) + require.Less(t, len(record.RawRequest), 1000, "the record must not scale with the body") + }) + + t.Run("an authenticated upload is not recorded as an interaction", func(t *testing.T) { + h, id, secret := uploadTestServer(t, true) + h.options.Auth = true + h.options.Token = "server-token" + + resp := probeUpload(t, h, payloadHost(id), "server-token", + uploadBody(t, id, secret, map[string][]byte{"evil.dtd": []byte("payload")})) + require.Equal(t, http.StatusOK, resp.StatusCode) + + require.Empty(t, storedInteractions(t, h, id), + "the operator's own upload must not be attributed to the target") + require.Zero(t, h.options.Stats.Http) + }) + + t.Run("a probe with no correlation id is dropped, not recorded", func(t *testing.T) { + h, id, _ := uploadTestServer(t, false) + + // No panic: handleInteraction slices a correlation id out of uniqueID. + resp := probeUpload(t, h, "example.com", "", payload) + require.Equal(t, http.StatusNotImplemented, resp.StatusCode) + require.Empty(t, storedInteractions(t, h, id)) + require.Zero(t, h.options.Stats.Http) + }) +} diff --git a/pkg/server/util.go b/pkg/server/util.go index f7313bb3..8cc15295 100644 --- a/pkg/server/util.go +++ b/pkg/server/util.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/projectdiscovery/gologger" + stringsutil "github.com/projectdiscovery/utils/strings" ) // Correlation ids are produced from xid (cidl prefix) and zbase32 (cidn suffix), @@ -85,3 +86,32 @@ func (options *Options) storeRootTLDInteraction(interaction *Interaction, id str gologger.Warning().Msgf("Could not store root tld %s interaction: %s\n", interaction.Protocol, err) } } + +// extractCorrelationID finds the first correlation id embedded in a host, +// returning the full unique id (correlation id plus nonce) and the host label +// prefix it was found in. +// +// This mirrors the extraction the logger middleware performs, so that a request +// served from the file route is attributed to the same session the logger would +// have attributed it to. TestExtractCorrelationIDMatchesLogger keeps the two in +// step. +func (options *Options) extractCorrelationID(host string) (uniqueID, fullID string) { + if hostOnly, _, err := net.SplitHostPort(host); err == nil { + host = hostOnly + } + parts := strings.Split(host, ".") + for i, part := range parts { + for chunk := range stringsutil.SlideWithLength(part, options.GetIdLength()) { + normalized := strings.ToLower(chunk) + if !options.isCorrelationID(normalized) { + continue + } + fullID := part + if i+1 <= len(parts) { + fullID = strings.Join(parts[:i+1], ".") + } + return normalized, fullID + } + } + return "", "" +} From fab656d336ef03975e57ed2a1216fe2a2c1e7550 Mon Sep 17 00:00:00 2001 From: nisay759 Date: Fri, 31 Jul 2026 00:11:00 +0200 Subject: [PATCH 06/20] server: serve hosted files from /f/ with a body-elided interaction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Files uploaded against a correlation id are now reachable at http(s)://./f/, and each fetch is recorded as an interaction so the tester sees the second stage fire. The route sits outside the logger middleware and records the interaction itself. Routing it through the logger instead would copy the response body into Interaction.RawResponse, which is then JSON-marshalled and appended to the session buffer -- a buffer with no cap in memory mode. jsoniter escapes each invalid UTF-8 byte as �, so a fetch of a 512KiB file of 0xff would retain roughly 3MB, from an unauthenticated GET, and the retained copy is mangled by that escaping anyway. TestServeUploadedFileElidesBody measures 2,935 bytes retained across five such fetches. Staying off defaultHandler also avoids three ways it would have been shadowed, each covered by a regression test: -dhr returns early for every request, the .json and .xml suffix branches would swallow payload.xml -- precisely the XXE case -- and -dr header injection could have stripped the forced Content-Type. Responses are always application/octet-stream with an attachment disposition and nosniff. DTD, XSLT and JNDI consumers ignore content type, so nothing is lost for the intended use, while the server never renders client-supplied HTML or SVG on its own domain. A file is only served to the session that owns it: the correlation id comes from the Host header via extractCorrelationID, which mirrors the logger's sliding-window extraction so serving and recording always agree on the session; TestExtractCorrelationIDMatchesLogger drives both implementations from one table so they cannot drift. The metadata record is consulted before touching disk, and the name is re-validated against the allowlist. Co-Authored-By: Claude Recording covers every exit, not only the successful one, through a single deferred call. A miss is evidence too: it is how the operator separates "the target never fetched the payload" from "the target asked for a name I am not hosting", or from a fetch arriving after the file expired. One call site rather than one per return, because this handler has six early exits and a seventh added later must not be able to drop the record silently. A hostedFetchRecorder passes writes through to the real ResponseWriter while noting the status and the bytes written, so the stored record states what was actually sent rather than assuming: ServeContent answers a conditional request with 304 and a ranged one with 206, and a record claiming 200 with the full length would assert a delivery that never happened. Wrapping the writer costs the io.ReaderFrom fast path in ServeContent's copy loop, which does not matter at the 1MiB default file cap. Two cases stay unrecorded because neither can be delivered: a host carrying no correlation id has nothing to be filed under, and a session that has left the cache has no bucket and no client polling it. So a fetch after deregistration is unrecoverable, while one after the file expired is recorded, the session outliving the file. Stats.Http is incremented where the interaction is recorded, so /metrics and the interaction stream cannot disagree about what arrived. Only the /f/ subtree is handled: CutPrefix rejects any other path instead of reading it as a file name. A bare /f is redirected to /f/ by ServeMux before this handler runs, so it is not recorded; the request that follows the redirect is. --- pkg/server/http_server.go | 5 + pkg/server/upload_handler.go | 173 +++++++++++++++++++++ pkg/server/upload_record_test.go | 162 +++++++++++++++++++ pkg/server/upload_serve_test.go | 256 +++++++++++++++++++++++++++++++ 4 files changed, 596 insertions(+) create mode 100644 pkg/server/upload_record_test.go create mode 100644 pkg/server/upload_serve_test.go diff --git a/pkg/server/http_server.go b/pkg/server/http_server.go index bc77744d..d55f809d 100644 --- a/pkg/server/http_server.go +++ b/pkg/server/http_server.go @@ -98,6 +98,11 @@ func NewHTTPServer(options *Options) (*HTTPServer, error) { // arriving here is a target probing the endpoint -- exactly what we exist to // record. router.Handle("/upload", server.corsMiddleware(http.HandlerFunc(server.uploadHandler))) + // Hosted files are served outside the logger middleware, which would + // otherwise copy each file body into an interaction record; the handler + // records a body-elided interaction itself. No CORS: these are fetched by + // the target under test, not cross-origin by a browser. + router.Handle("/f/", http.HandlerFunc(server.serveUploadedFile)) if server.options.EnableMetrics { router.Handle("/metrics", server.corsMiddleware(server.authMiddleware(http.HandlerFunc(server.metricsHandler)))) } diff --git a/pkg/server/upload_handler.go b/pkg/server/upload_handler.go index 0429ec0e..4320a1e8 100644 --- a/pkg/server/upload_handler.go +++ b/pkg/server/upload_handler.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "mime" "net" "net/http" "net/http/httputil" @@ -238,6 +239,178 @@ func (h *HTTPServer) uploadHandler(w http.ResponseWriter, req *http.Request) { _ = json.NewEncoder(w).Encode(&UploadResponse{Message: "upload successful", Files: response}) } +// serveUploadedFile serves a file hosted against the correlation id in the Host +// header, and records the fetch as an interaction. +// +// This runs on its own route, outside the logger middleware, and records the +// interaction itself. The logger buffers the whole response into a recorder and +// dumps the body into Interaction.RawResponse; for a hosted file that means +// every fetch retains a multiple of the file size in the session's interaction +// buffer, which has no cap in memory mode, and the retained copy is mangled by +// JSON escaping anyway. Recording explicitly keeps the evidence that the fetch +// happened -- which is the entire point for second-stage OOB verification -- +// without the payload. +// +// Serving here rather than from defaultHandler also avoids being shadowed by +// -dhr, by the .json/.xml suffix branches, and by -dr header injection. +func (h *HTTPServer) serveUploadedFile(w http.ResponseWriter, req *http.Request) { + // Every exit below is recorded, by one deferred call rather than a call per + // return: a miss is the operator's evidence that the target fetched *some* + // path -- the wrong name, or a file that has since expired -- and adding an + // early return here must not be able to silently drop that again. + rec := &hostedFetchRecorder{ResponseWriter: w} + uniqueID, fullID := h.options.extractCorrelationID(req.Host) + var meta *storage.UploadedFile + defer func() { h.recordHostedFetch(req, uniqueID, fullID, meta, rec) }() + + store := h.options.UploadStore + uploadStorage := h.options.UploadStorage() + if store == nil || uploadStorage == nil { + http.NotFound(rec, req) + return + } + if uniqueID == "" { + http.NotFound(rec, req) + return + } + correlationID := uniqueID[:h.options.CorrelationIdLength] + + // Only the /f/ subtree is ours. CutPrefix rather than TrimPrefix so that a + // path which does not carry the prefix is rejected instead of being read as + // a file name; ServeMux redirects a bare /f to /f/ before we are reached. + name, ok := strings.CutPrefix(req.URL.Path, "/f/") + if !ok || !isSafeUploadName(name) { + http.NotFound(rec, req) + return + } + + // Consult the metadata first: it makes a miss cheap, and means a file can + // only be served to the session that actually owns it. + files, ok := uploadStorage.ListUploads(correlationID) + if !ok { + http.NotFound(rec, req) + return + } + for i := range files { + if files[i].Name == name { + meta = &files[i] + break + } + } + if meta == nil { + http.NotFound(rec, req) + return + } + + f, fi, err := store.Open(correlationID, name) + if err != nil { + gologger.Debug().Msgf("Could not open uploaded file %s/%s: %s\n", correlationID, name, err) + meta = nil // served nothing, so the record must not claim a hit + http.NotFound(rec, req) + return + } + defer f.Close() + + // Always octet-stream with an attachment disposition: DTD, XSLT and JNDI + // consumers ignore content type entirely, so nothing is lost for the + // intended use, while the server never renders client-supplied HTML or SVG + // on its own domain. + rec.Header().Set("Content-Type", "application/octet-stream") + rec.Header().Set("Content-Disposition", mime.FormatMediaType("attachment", map[string]string{"filename": name})) + rec.Header().Set("X-Content-Type-Options", "nosniff") + if !h.options.NoVersionHeader { + rec.Header().Set("X-Interactsh-Version", h.options.Version) + } + + // ServeContent handles Range and conditional requests. The empty name + // argument keeps it from re-deriving a content type from the extension. + http.ServeContent(rec, req, "", fi.ModTime(), f) +} + +// hostedFetchRecorder passes writes straight through to the real +// ResponseWriter while noting what the response actually was, so the stored +// interaction can state it rather than assume it. ServeContent answers a +// conditional request with 304 and a ranged one with 206, and a record that +// claimed 200 with the full length would be evidence of a delivery that never +// happened. +type hostedFetchRecorder struct { + http.ResponseWriter + status int + written int64 +} + +func (r *hostedFetchRecorder) WriteHeader(code int) { + if r.status == 0 { + r.status = code + } + r.ResponseWriter.WriteHeader(code) +} + +func (r *hostedFetchRecorder) Write(b []byte) (int, error) { + if r.status == 0 { + r.status = http.StatusOK + } + n, err := r.ResponseWriter.Write(b) + r.written += int64(n) + return n, err +} + +// statusCode reports the status sent, defaulting to 200 for a handler that +// wrote neither a header nor a body. +func (r *hostedFetchRecorder) statusCode() int { + if r.status == 0 { + return http.StatusOK + } + return r.status +} + +// recordHostedFetch stores an interaction for any request to the /f/ subtree, +// hit or miss, with the response body replaced by a summary. +// +// A miss matters as much as a hit: it is how the operator tells "the target +// never fetched the payload" from "the target fetched a name I am not hosting", +// or from a fetch that arrived after the file had expired. meta is nil for +// every miss, which is what selects the summary. +func (h *HTTPServer) recordHostedFetch(req *http.Request, uniqueID, fullID string, meta *storage.UploadedFile, rec *hostedFetchRecorder) { + if uniqueID == "" { + // Nothing to attribute it to: interactions are indexed by correlation id, + // and handleInteraction slices one out of uniqueID unconditionally. + return + } + // Counted where it is recorded, so /metrics and the interaction stream + // cannot disagree about what arrived. + atomic.AddUint64(&h.options.Stats.Http, 1) + + var host string + if originIP := req.Header.Get(h.options.OriginIPHeader); originIP != "" { + host = originIP + } else { + host, _, _ = net.SplitHostPort(req.RemoteAddr) + } + + // Request dumped without its body: a file fetch is a GET, and an attacker + // controlling the body must not be able to inflate the stored record. + reqDump, _ := httputil.DumpRequest(req, false) + + status := rec.statusCode() + var resp strings.Builder + fmt.Fprintf(&resp, "HTTP/1.1 %d %s\r\n", status, http.StatusText(status)) + if meta == nil { + fmt.Fprintf(&resp, "\r\n[no hosted file for %q on this session]\n", req.URL.Path) + h.handleInteraction(req, uniqueID, fullID, string(reqDump), resp.String(), host) + return + } + fmt.Fprintf(&resp, "Content-Type: application/octet-stream\r\n") + fmt.Fprintf(&resp, "Content-Disposition: attachment; filename=%q\r\n", meta.Name) + fmt.Fprintf(&resp, "Content-Length: %d\r\n\r\n", rec.written) + // The byte count is what ServeContent actually wrote, so a 304 records as + // zero bytes and a 206 as the size of the range. + fmt.Fprintf(&resp, "[body elided: %d of %d bytes of uploaded file %q, sha256 %s]\n", + rec.written, meta.Size, meta.Name, meta.SHA256) + + h.handleInteraction(req, uniqueID, fullID, string(reqDump), resp.String(), host) +} + // recordUploadProbe records an interaction for a request to /upload that no // legitimate client could have sent, so that a target poking at the endpoint // reaches the operator's stream instead of vanishing behind a 401 or a 501. diff --git a/pkg/server/upload_record_test.go b/pkg/server/upload_record_test.go new file mode 100644 index 00000000..120e28c2 --- /dev/null +++ b/pkg/server/upload_record_test.go @@ -0,0 +1,162 @@ +package server + +import ( + "fmt" + "net/http" + "net/http/httptest" + "testing" + + jsoniter "github.com/json-iterator/go" + "github.com/stretchr/testify/require" +) + +// serveRequestWithHeaders is serveRequest with request headers, for the +// conditional and ranged cases ServeContent handles on its own. +func serveRequestWithHeaders(t *testing.T, h *HTTPServer, host, path string, hdr map[string]string) *http.Response { + t.Helper() + req := httptest.NewRequest(http.MethodGet, path, nil) + req.Host = host + req.RemoteAddr = "203.0.113.7:5555" + for k, v := range hdr { + req.Header.Set(k, v) + } + w := httptest.NewRecorder() + h.serveUploadedFile(w, req) + return w.Result() +} + +// records returns the interaction records stored for a session, newest last. +func records(t *testing.T, h *HTTPServer, correlationID string) []*Interaction { + t.Helper() + item, err := h.options.Storage.GetCacheItem(correlationID) + require.NoError(t, err) + out := make([]*Interaction, 0, len(item.Data)) + for _, raw := range item.Data { + record := &Interaction{} + require.NoError(t, jsoniter.Unmarshal([]byte(raw), record)) + out = append(out, record) + } + return out +} + +// A fetch that misses is evidence too: it separates "the target never came" from +// "the target asked for a name I am not hosting". +func TestServeUploadedFileRecordsMisses(t *testing.T) { + setup := func(t *testing.T) (*HTTPServer, string) { + t.Helper() + h, id, secret := uploadTestServer(t, true) + require.Equal(t, http.StatusOK, + doUpload(t, h, uploadBody(t, id, secret, map[string][]byte{"evil.dtd": []byte("payload")})).StatusCode) + return h, id + } + + t.Run("a name that is not hosted", func(t *testing.T) { + h, id := setup(t) + resp := serveRequestWithHeaders(t, h, payloadHost(id), "/f/evil.dtd.txt", nil) + require.Equal(t, http.StatusNotFound, resp.StatusCode) + + got := records(t, h, id) + require.Len(t, got, 1, "a miss must reach the operator") + require.Contains(t, got[0].RawRequest, "GET /f/evil.dtd.txt") + require.Contains(t, got[0].RawResponse, "404 Not Found") + require.Contains(t, got[0].RawResponse, `[no hosted file for "/f/evil.dtd.txt" on this session]`) + require.NotContains(t, got[0].RawResponse, "body elided", "nothing was served") + }) + + t.Run("a rejected file name", func(t *testing.T) { + h, id := setup(t) + require.Equal(t, http.StatusNotFound, + serveRequestWithHeaders(t, h, payloadHost(id), "/f/../../etc/passwd", nil).StatusCode) + require.Len(t, records(t, h, id), 1) + }) + + t.Run("a server with uploads disabled", func(t *testing.T) { + h, id, _ := uploadTestServer(t, false) + require.Equal(t, http.StatusNotFound, + serveRequestWithHeaders(t, h, payloadHost(id), "/f/evil.dtd", nil).StatusCode) + + got := records(t, h, id) + require.Len(t, got, 1, "a probe of /f/ is worth recording even with uploads off") + require.Contains(t, got[0].RawResponse, "404 Not Found") + }) + + t.Run("a host carrying no correlation id is dropped", func(t *testing.T) { + h, id := setup(t) + require.Equal(t, http.StatusNotFound, + serveRequestWithHeaders(t, h, "example.com", "/f/evil.dtd", nil).StatusCode) + require.Empty(t, records(t, h, id)) + require.Zero(t, h.options.Stats.Http, "nothing recorded, so nothing counted") + }) +} + +// The stored record must state what ServeContent actually sent. Claiming a full +// 200 for a 304 is evidence of a delivery that never happened. +func TestServeUploadedFileRecordsActualResponse(t *testing.T) { + content := []byte("PAYLOAD") + setup := func(t *testing.T) (*HTTPServer, string) { + t.Helper() + h, id, secret := uploadTestServer(t, true) + require.Equal(t, http.StatusOK, + doUpload(t, h, uploadBody(t, id, secret, map[string][]byte{"evil.dtd": content})).StatusCode) + return h, id + } + + t.Run("a conditional request answered 304 records zero bytes", func(t *testing.T) { + h, id := setup(t) + first := serveRequestWithHeaders(t, h, payloadHost(id), "/f/evil.dtd", nil) + lastMod := first.Header.Get("Last-Modified") + require.NotEmpty(t, lastMod) + + resp := serveRequestWithHeaders(t, h, payloadHost(id), "/f/evil.dtd", + map[string]string{"If-Modified-Since": lastMod}) + require.Equal(t, http.StatusNotModified, resp.StatusCode) + + got := records(t, h, id) + require.Len(t, got, 2) + require.Contains(t, got[1].RawResponse, "304 Not Modified") + require.Contains(t, got[1].RawResponse, fmt.Sprintf("[body elided: 0 of %d bytes", len(content))) + require.NotContains(t, got[1].RawResponse, "200 OK") + }) + + t.Run("a ranged request answered 206 records the range size", func(t *testing.T) { + h, id := setup(t) + resp := serveRequestWithHeaders(t, h, payloadHost(id), "/f/evil.dtd", + map[string]string{"Range": "bytes=0-1"}) + require.Equal(t, http.StatusPartialContent, resp.StatusCode) + + got := records(t, h, id) + require.Len(t, got, 1) + require.Contains(t, got[0].RawResponse, "206 Partial Content") + require.Contains(t, got[0].RawResponse, fmt.Sprintf("[body elided: 2 of %d bytes", len(content))) + }) + + t.Run("a plain fetch still records the full length", func(t *testing.T) { + h, id := setup(t) + require.Equal(t, http.StatusOK, + serveRequestWithHeaders(t, h, payloadHost(id), "/f/evil.dtd", nil).StatusCode) + + got := records(t, h, id) + require.Len(t, got, 1) + require.Contains(t, got[0].RawResponse, "200 OK") + require.Contains(t, got[0].RawResponse, fmt.Sprintf("Content-Length: %d", len(content))) + require.Contains(t, got[0].RawResponse, fmt.Sprintf("[body elided: %d of %d bytes", len(content), len(content))) + }) +} + +// /metrics and the interaction stream must not disagree about what arrived. +func TestServeUploadedFileMetricsMatchRecords(t *testing.T) { + h, id, secret := uploadTestServer(t, true) + require.Equal(t, http.StatusOK, + doUpload(t, h, uploadBody(t, id, secret, map[string][]byte{"evil.dtd": []byte("payload")})).StatusCode) + + // two hits, two misses, and one request that cannot be attributed + serveRequestWithHeaders(t, h, payloadHost(id), "/f/evil.dtd", nil) + serveRequestWithHeaders(t, h, payloadHost(id), "/f/evil.dtd", nil) + serveRequestWithHeaders(t, h, payloadHost(id), "/f/absent.dtd", nil) + serveRequestWithHeaders(t, h, payloadHost(id), "/f/also-absent.dtd", nil) + serveRequestWithHeaders(t, h, "example.com", "/f/evil.dtd", nil) + + require.Len(t, records(t, h, id), 4) + require.Equal(t, uint64(4), h.options.Stats.Http, + "the http counter must equal the interactions actually recorded") +} diff --git a/pkg/server/upload_serve_test.go b/pkg/server/upload_serve_test.go new file mode 100644 index 00000000..1eda44a1 --- /dev/null +++ b/pkg/server/upload_serve_test.go @@ -0,0 +1,256 @@ +package server + +import ( + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "strings" + "testing" + + jsoniter "github.com/json-iterator/go" + "github.com/projectdiscovery/interactsh/pkg/storage" + stringsutil "github.com/projectdiscovery/utils/strings" + "github.com/stretchr/testify/require" +) + +// serveRequest issues a GET for a hosted file against the given host. +func serveRequest(t *testing.T, h *HTTPServer, host, path string) *http.Response { + t.Helper() + // Path-only target, so the dumped request line matches what a real server + // sees rather than an absolute-URI form. + req := httptest.NewRequest(http.MethodGet, path, nil) + req.Host = host + req.RemoteAddr = "203.0.113.7:5555" + w := httptest.NewRecorder() + h.serveUploadedFile(w, req) + return w.Result() +} + +func TestServeUploadedFile(t *testing.T) { + content := []byte(``) + + setup := func(t *testing.T, name string) (*HTTPServer, string, string) { + t.Helper() + h, id, secret := uploadTestServer(t, true) + resp := doUpload(t, h, uploadBody(t, id, secret, map[string][]byte{name: content})) + require.Equal(t, http.StatusOK, resp.StatusCode) + return h, id, secret + } + + t.Run("serves the exact bytes with hardened headers", func(t *testing.T) { + h, id, _ := setup(t, "evil.dtd") + + resp := serveRequest(t, h, payloadHost(id), "/f/evil.dtd") + require.Equal(t, http.StatusOK, resp.StatusCode) + + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + require.Equal(t, content, body) + + require.Equal(t, "application/octet-stream", resp.Header.Get("Content-Type")) + require.Equal(t, `attachment; filename=evil.dtd`, resp.Header.Get("Content-Disposition")) + require.Equal(t, "nosniff", resp.Header.Get("X-Content-Type-Options")) + }) + + // The old design served from defaultHandler, where these two branches + // would have hijacked the request and returned reflection markup. + t.Run("xml and json names are not hijacked", func(t *testing.T) { + for _, name := range []string{"payload.xml", "payload.json"} { + h, id, _ := setup(t, name) + resp := serveRequest(t, h, payloadHost(id), "/f/"+name) + require.Equal(t, http.StatusOK, resp.StatusCode) + + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + require.Equal(t, content, body, "%s must return the file, not reflected markup", name) + } + }) + + // -dhr returns early for every request inside defaultHandler. + t.Run("default http response file does not shadow it", func(t *testing.T) { + h, id, _ := setup(t, "evil.dtd") + h.defaultResponse = "default response" + + resp := serveRequest(t, h, payloadHost(id), "/f/evil.dtd") + require.Equal(t, http.StatusOK, resp.StatusCode) + body, _ := io.ReadAll(resp.Body) + require.Equal(t, content, body) + }) + + // -dr lets query parameters inject headers on the /s/ static path. + t.Run("dynamic response cannot override the hardened headers", func(t *testing.T) { + h, id, _ := setup(t, "evil.dtd") + h.options.DynamicResp = true + + resp := serveRequest(t, h, payloadHost(id), "/f/evil.dtd?header=Content-Type:text/html&status=201") + require.Equal(t, http.StatusOK, resp.StatusCode, "status must not be attacker controlled") + require.Equal(t, "application/octet-stream", resp.Header.Get("Content-Type")) + }) + + t.Run("404s", func(t *testing.T) { + h, id, _ := setup(t, "evil.dtd") + + t.Run("host without a correlation id", func(t *testing.T) { + resp := serveRequest(t, h, "www.oast.test", "/f/evil.dtd") + require.Equal(t, http.StatusNotFound, resp.StatusCode) + }) + t.Run("unknown file", func(t *testing.T) { + resp := serveRequest(t, h, payloadHost(id), "/f/absent.dtd") + require.Equal(t, http.StatusNotFound, resp.StatusCode) + }) + t.Run("traversal", func(t *testing.T) { + resp := serveRequest(t, h, payloadHost(id), "/f/../../etc/passwd") + require.Equal(t, http.StatusNotFound, resp.StatusCode) + }) + t.Run("file owned by a different session", func(t *testing.T) { + other, otherID, otherSecret := uploadTestServer(t, true) + r := doUpload(t, other, uploadBody(t, otherID, otherSecret, map[string][]byte{"secret.dtd": []byte("theirs")})) + require.Equal(t, http.StatusOK, r.StatusCode) + + // Ask our server, using our session, for their file name. + resp := serveRequest(t, h, payloadHost(id), "/f/secret.dtd") + require.Equal(t, http.StatusNotFound, resp.StatusCode) + }) + t.Run("uploads disabled", func(t *testing.T) { + off, offID, _ := uploadTestServer(t, false) + resp := serveRequest(t, off, payloadHost(offID), "/f/evil.dtd") + require.Equal(t, http.StatusNotFound, resp.StatusCode) + }) + }) +} + +func TestServeUploadedFileRecordsInteraction(t *testing.T) { + h, id, secret := uploadTestServer(t, true) + content := []byte("dtd content") + require.Equal(t, http.StatusOK, + doUpload(t, h, uploadBody(t, id, secret, map[string][]byte{"evil.dtd": content})).StatusCode) + + resp := serveRequest(t, h, payloadHost(id), "/f/evil.dtd") + require.Equal(t, http.StatusOK, resp.StatusCode) + + // Read the raw buffer rather than GetInteractions: in memory mode the + // latter encrypts on the way out, and we want to inspect what was stored. + item, err := h.options.Storage.GetCacheItem(id) + require.NoError(t, err) + require.Len(t, item.Data, 1, "the fetch must be visible to the client that owns the session") + + record := &Interaction{} + require.NoError(t, jsoniter.Unmarshal([]byte(item.Data[0]), record)) + + require.Equal(t, "http", record.Protocol) + require.Contains(t, record.RawRequest, "GET /f/evil.dtd") + require.Equal(t, "203.0.113.7", record.RemoteAddress) + require.Contains(t, record.RawResponse, "200 OK") + require.Contains(t, record.RawResponse, "body elided") +} + +// The reason serving sits outside the logger middleware: routed through it, a +// fetch would retain a multiple of the file size in the session's interaction +// buffer. +func TestServeUploadedFileElidesBody(t *testing.T) { + h, id, secret := uploadTestServer(t, true) + h.options.UploadStore.maxFileSize = 1 << 20 + + // Bytes that JSON-escape badly, which is what drove the amplification. + content := make([]byte, 512*1024) + for i := range content { + content[i] = 0xff + } + require.Equal(t, http.StatusOK, + doUpload(t, h, uploadBody(t, id, secret, map[string][]byte{"big.bin": content})).StatusCode) + + for i := 0; i < 5; i++ { + require.Equal(t, http.StatusOK, serveRequest(t, h, payloadHost(id), "/f/big.bin").StatusCode) + } + + item, err := h.options.Storage.GetCacheItem(id) + require.NoError(t, err) + require.Len(t, item.Data, 5) + + var total int + for _, raw := range item.Data { + total += len(raw) + require.NotContains(t, raw, strings.Repeat("\\ufffd", 64), + "file bytes must not be copied into the interaction record") + } + t.Logf("5 fetches of a 512KiB file retained %d bytes of interaction data", total) + require.Less(t, total, 32*1024, + fmt.Sprintf("5 fetches of a 512KiB file retained %d bytes; body elision is not working", total)) +} + +func TestExtractCorrelationIDMatchesLogger(t *testing.T) { + options := &Options{CorrelationIdLength: 20, CorrelationIdNonceLength: 13} + id := "c6rj61aciaeutn2ae680" + nonce := "xk4tqy8pqhwty" + + cases := []string{ + id + nonce + ".oast.test", + id + nonce + ".sub.oast.test", + strings.ToUpper(id+nonce) + ".oast.test", + id + nonce + ".oast.test:8080", + "www.oast.test", + "", + id + ".oast.test", // too short to be a full unique id + } + + for _, host := range cases { + wantUnique, wantFull := loggerStyleExtract(options, host) + gotUnique, gotFull := options.extractCorrelationID(host) + require.Equal(t, wantUnique, gotUnique, "unique id mismatch for host %q", host) + require.Equal(t, wantFull, gotFull, "full id mismatch for host %q", host) + } +} + +// loggerStyleExtract reproduces the extraction the logger middleware performs, +// so the two implementations can be compared directly. +func loggerStyleExtract(options *Options, host string) (string, string) { + if hostOnly, _, err := net.SplitHostPort(host); err == nil { + host = hostOnly + } + parts := strings.Split(host, ".") + for i, part := range parts { + for partChunk := range stringsutil.SlideWithLength(part, options.GetIdLength()) { + normalized := strings.ToLower(partChunk) + if options.isCorrelationID(normalized) { + fullID := part + if i+1 <= len(parts) { + fullID = strings.Join(parts[:i+1], ".") + } + return normalized, fullID + } + } + } + return "", "" +} + +// noUploadStorage is a Storage without the UploadStorage capability, standing in +// for a shared backend such as Redis. Embedding the interface supplies every +// Storage method while deliberately omitting UpdateUploads and ListUploads. +type noUploadStorage struct{ storage.Storage } + +// TestUploadsDegradeWhenBackendCannotTrackThem covers the path a deployment on a +// shared storage backend would take. The server must decline uploads outright +// rather than advertise a capability it cannot honour, or panic reaching for a +// nil capability. +func TestUploadsDegradeWhenBackendCannotTrackThem(t *testing.T) { + h, id, secret := uploadTestServer(t, true) + require.True(t, h.capabilities().Upload, "precondition: a capable backend advertises uploads") + + resp := doUpload(t, h, uploadBody(t, id, secret, map[string][]byte{"evil.dtd": []byte("payload")})) + require.Equal(t, http.StatusOK, resp.StatusCode, "precondition: upload works before swapping the backend") + + // Swap in a backend that cannot track uploads. The files stay on disk, so + // this isolates the capability check from the storage contents. + h.options.Storage = noUploadStorage{h.options.Storage} + require.Nil(t, h.options.UploadStorage(), "backend must not satisfy UploadStorage") + + require.False(t, h.capabilities().Upload, "capability must not be advertised without backend support") + + resp = doUpload(t, h, uploadBody(t, id, secret, map[string][]byte{"other.dtd": []byte("x")})) + require.Equal(t, http.StatusNotImplemented, resp.StatusCode, "upload must report unsupported, not fail late") + + served := serveRequest(t, h, payloadHost(id), "/f/evil.dtd") + require.Equal(t, http.StatusNotFound, served.StatusCode, "serving must 404 rather than dereference a nil capability") +} From 0d57c82fae4f8dfc9acd59bd04765ba9fab1175c Mon Sep 17 00:00:00 2001 From: nisay759 Date: Fri, 31 Jul 2026 00:13:18 +0200 Subject: [PATCH 07/20] server: hide hosted files from FTP listings, attribute downloads Two changes to make FTP a safe way to serve hosted files. NopAuth accepts any credentials, and NopDriver forwards ListDir to the real file driver. Once the upload root is also the FTP root, that combination lets an anonymous client enumerate every correlation id that currently has hosted files and then walk into each one. Sessions live under a single .interactsh-user-uploads directory, so ListDir closes that off with two rules: that directory never lists its own contents, and it is filtered out of the root listing so it cannot be discovered in the first place. Both go through one path.Clean-based helper, so the //, /./ and /x/../ spellings cannot slip past, and the same guard covers NLST, MLSD and STAT, which all route through ListDir. A client that knows its own correlation id can still list inside it, and RETR by full path is untouched. Deliberately not a blanket refusal on the root: -ftp-dir is documented as listing the operator's own directory in read-only mode, and refusing the root silently broke that for every -ftp user, whether or not -upload was enabled. FTP interactions were all stored under options.Token, the shared bucket that pollHandler fans out to every authenticated client. That is reasonable for connection noise, but a fetch of one session's hosted DTD would be reported to everybody and attributed to nobody. The download hooks now derive the correlation id from the path segment inside .interactsh-user-uploads, verify it names a live session with uploads, and record against that session instead. The path is cleaned first, so traversal can only resolve to the session it actually points at, and a path anywhere else under the FTP root is the operator's rather than ours and is never attributed. Everything unattributable -- logins, directory changes, downloads outside any session -- keeps its existing behaviour. Co-Authored-By: Claude --- pkg/server/ftp_server.go | 109 +++++++++++++++++- pkg/server/ftp_upload_test.go | 201 ++++++++++++++++++++++++++++++++++ 2 files changed, 305 insertions(+), 5 deletions(-) create mode 100644 pkg/server/ftp_upload_test.go diff --git a/pkg/server/ftp_server.go b/pkg/server/ftp_server.go index 79a68f59..860c24a0 100644 --- a/pkg/server/ftp_server.go +++ b/pkg/server/ftp_server.go @@ -5,11 +5,14 @@ import ( "fmt" "io" "os" + "path" "strings" "sync/atomic" "time" "encoding/json" + + "github.com/asaskevich/govalidator" "github.com/projectdiscovery/gologger" ftpserver "goftp.io/server/v2" "goftp.io/server/v2/driver/file" @@ -119,6 +122,18 @@ func (h *FTPServer) Close() { } func (h *FTPServer) recordInteraction(remoteAddress, data string) { + h.recordInteractionForPath(remoteAddress, data, "") +} + +// recordInteractionForPath records an FTP interaction, attributing it to the +// session that owns dstPath when the path points into a hosted-files directory. +// +// Interactions that cannot be attributed go, as before, to the shared token +// bucket, which pollHandler fans out to every authenticated client. That is +// fine for connection noise, but a fetch of a specific session's hosted file +// belongs to that session: otherwise it is reported to everyone and attributed +// to no one. +func (h *FTPServer) recordInteractionForPath(remoteAddress, data, dstPath string) { atomic.AddUint64(&h.options.Stats.Ftp, 1) if data == "" { @@ -130,15 +145,54 @@ func (h *FTPServer) recordInteraction(remoteAddress, data string) { RawRequest: data, Timestamp: time.Now(), } + + correlationID := h.correlationIDFromPath(dstPath) + if correlationID != "" { + interaction.UniqueID = correlationID + interaction.FullId = correlationID + } + dataBytes, err := json.Marshal(interaction) if err != nil { gologger.Warning().Msgf("Could not encode ftp interaction: %s\n", err) - } else { - gologger.Debug().Msgf("FTP Interaction: \n%s\n", string(dataBytes)) - if err := h.options.Storage.AddInteractionWithId(h.options.Token, dataBytes); err != nil { + return + } + gologger.Debug().Msgf("FTP Interaction: \n%s\n", string(dataBytes)) + + if correlationID != "" { + if err := h.options.Storage.AddInteraction(correlationID, dataBytes); err != nil { gologger.Warning().Msgf("Could not store ftp interaction: %s\n", err) } + return + } + if err := h.options.Storage.AddInteractionWithId(h.options.Token, dataBytes); err != nil { + gologger.Warning().Msgf("Could not store ftp interaction: %s\n", err) + } +} + +// correlationIDFromPath returns the correlation id owning an FTP path, or "" if +// the path does not name a live session with hosted files. +func (h *FTPServer) correlationIDFromPath(dstPath string) string { + uploadStorage := h.options.UploadStorage() + if dstPath == "" || h.options.UploadStore == nil || uploadStorage == nil { + return "" } + // Hosted files live at ///, so the + // correlation id is the second segment. Cleaning first means a traversal can + // only ever resolve to the session it actually points at. + rest, ok := strings.CutPrefix(ftpCleanPath(dstPath), "/"+uploadsDirName+"/") + if !ok { + return "" + } + segment, _, _ := strings.Cut(rest, "/") + segment = strings.ToLower(segment) + if len(segment) != h.options.CorrelationIdLength || !govalidator.IsAlphanumeric(segment) { + return "" + } + if _, ok := uploadStorage.ListUploads(segment); !ok { + return "" + } + return segment } func (h *FTPServer) Print(sessionID string, message interface{}) {} @@ -211,7 +265,7 @@ func (h *FTPServer) BeforeDownloadFile(ctx *ftpserver.Context, dstPath string) { b.WriteString(ctx.Param) b.WriteString("\n") b.WriteString("downloading file " + dstPath) - h.recordInteraction(ctx.Sess.RemoteAddr().String(), b.String()) + h.recordInteractionForPath(ctx.Sess.RemoteAddr().String(), b.String(), dstPath) } func (h *FTPServer) AfterUserLogin(ctx *ftpserver.Context, userName, password string, passMatched bool, err error) { var b strings.Builder @@ -247,7 +301,7 @@ func (h *FTPServer) AfterFileDownloaded(ctx *ftpserver.Context, dstPath string, b.WriteString(ctx.Param) b.WriteString("\n") b.WriteString("downloaded file " + dstPath) - h.recordInteraction(ctx.Sess.RemoteAddr().String(), b.String()) + h.recordInteractionForPath(ctx.Sess.RemoteAddr().String(), b.String(), dstPath) } func (h *FTPServer) AfterCurDirChanged(ctx *ftpserver.Context, oldCurDir, newCurDir string, err error) { var b strings.Builder @@ -295,10 +349,55 @@ func (n *NopDriver) Stat(c *ftpserver.Context, s string) (os.FileInfo, error) { return n.driver.Stat(c, s) } +// ListDir hides interactsh's own upload storage, and nothing else. +// +// NopAuth accepts any credentials, so an anonymous client must not be able to +// read off every correlation id that currently has uploaded files and then walk +// into each one. Two rules close that off: the uploads directory never lists its +// own contents, and it is filtered out of the root listing so that it cannot be +// discovered in the first place. A client that already knows its own correlation +// id can still list and RETR inside it. +// +// Only those two rules, deliberately: the root itself lists normally, because +// -ftp-dir is documented as serving the operator's own directory and a blanket +// refusal there would silently break that. func (n *NopDriver) ListDir(c *ftpserver.Context, s string, f func(os.FileInfo) error) error { + if isUploadsDir(s) { + return nil + } + if isFTPRoot(s) { + return n.driver.ListDir(c, s, func(info os.FileInfo) error { + if info.Name() == uploadsDirName { + return nil + } + return f(info) + }) + } return n.driver.ListDir(c, s, f) } +// isUploadsDir reports whether an FTP path refers to the uploads directory +// itself, which sits directly under the root. +func isUploadsDir(p string) bool { + return ftpCleanPath(p) == "/"+uploadsDirName +} + +// isFTPRoot reports whether an FTP path refers to the server root. +func isFTPRoot(p string) bool { + switch ftpCleanPath(p) { + case "/", ".", "": + return true + } + return false +} + +// ftpCleanPath resolves an FTP path to an absolute, traversal-free form. Every +// path decision in this file goes through it, so "/a/../b", "//b" and "/./b" can +// never be treated differently from "/b". +func ftpCleanPath(p string) string { + return path.Clean("/" + strings.TrimPrefix(p, "/")) +} + func (n *NopDriver) DeleteDir(c *ftpserver.Context, s string) error { return nil } diff --git a/pkg/server/ftp_upload_test.go b/pkg/server/ftp_upload_test.go new file mode 100644 index 00000000..31dd0862 --- /dev/null +++ b/pkg/server/ftp_upload_test.go @@ -0,0 +1,201 @@ +package server + +import ( + "net/http" + "os" + "testing" + "time" + + jsoniter "github.com/json-iterator/go" + "github.com/stretchr/testify/require" + ftpserver "goftp.io/server/v2" +) + +func TestIsFTPRoot(t *testing.T) { + for _, p := range []string{"/", "", ".", "//", "/.", "/../.."} { + require.True(t, isFTPRoot(p), "expected %q to be treated as root", p) + } + for _, p := range []string{"/c6rj61aciaeutn2ae680", "/a/b", "/evil.dtd", "/dir/"} { + require.False(t, isFTPRoot(p), "expected %q not to be treated as root", p) + } +} + +// countingDriver records which paths reached the wrapped driver, and reports the +// entries it was configured with for each of them. +type countingDriver struct { + ftpserver.Driver + listed []string + entries []string +} + +func (d *countingDriver) ListDir(c *ftpserver.Context, s string, f func(os.FileInfo) error) error { + d.listed = append(d.listed, s) + for _, name := range d.entries { + if err := f(dirEntryInfo(name)); err != nil { + return err + } + } + return nil +} + +// dirEntryInfo is a minimal os.FileInfo standing in for a directory entry. +type dirEntryInfo string + +func (d dirEntryInfo) Name() string { return string(d) } +func (d dirEntryInfo) Size() int64 { return 0 } +func (d dirEntryInfo) Mode() os.FileMode { return os.ModeDir | 0o700 } +func (d dirEntryInfo) ModTime() time.Time { return time.Time{} } +func (d dirEntryInfo) IsDir() bool { return true } +func (d dirEntryInfo) Sys() interface{} { return nil } + +func TestIsUploadsDir(t *testing.T) { + for _, p := range []string{ + "/" + uploadsDirName, + "/" + uploadsDirName + "/", + "//" + uploadsDirName, + "/./" + uploadsDirName, + "/other/../" + uploadsDirName, + "/" + uploadsDirName + "/.", + } { + require.True(t, isUploadsDir(p), "expected %q to resolve to the uploads directory", p) + } + for _, p := range []string{ + "/", + "/" + uploadsDirName + "/c6rj61aciaeutn2ae680", + "/nested/" + uploadsDirName, + "/" + uploadsDirName + "-other", + } { + require.False(t, isUploadsDir(p), "expected %q not to resolve to the uploads directory", p) + } +} + +// NopAuth accepts any credentials, so an unauthenticated client must not be able +// to enumerate the correlation ids that currently have hosted files. +func TestNopDriverHidesUploadsDir(t *testing.T) { + t.Run("the uploads directory never enumerates", func(t *testing.T) { + for _, p := range []string{ + "/" + uploadsDirName, + "/" + uploadsDirName + "/", + "//" + uploadsDirName, + "/./" + uploadsDirName, + "/pub/../" + uploadsDirName, + } { + inner := &countingDriver{entries: []string{"c6rj61aciaeutn2ae680", "c6rj61aciaeutn2ae681"}} + driver := NewNopDriver(inner) + + var seen []string + require.NoError(t, driver.ListDir(nil, p, func(fi os.FileInfo) error { + seen = append(seen, fi.Name()) + return nil + })) + require.Empty(t, seen, "%q must not enumerate correlation ids", p) + require.Empty(t, inner.listed, "%q must not even reach the filesystem", p) + } + }) + + t.Run("the root lists operator content without the uploads directory", func(t *testing.T) { + inner := &countingDriver{entries: []string{"index.html", uploadsDirName, "assets"}} + driver := NewNopDriver(inner) + + var seen []string + require.NoError(t, driver.ListDir(nil, "/", func(fi os.FileInfo) error { + seen = append(seen, fi.Name()) + return nil + })) + require.Equal(t, []string{"index.html", "assets"}, seen, + "-ftp-dir is documented as serving the operator's directory, minus our own") + }) + + t.Run("a session directory still lists for a client that knows its id", func(t *testing.T) { + inner := &countingDriver{entries: []string{"evil.dtd"}} + driver := NewNopDriver(inner) + p := "/" + uploadsDirName + "/c6rj61aciaeutn2ae680" + + var seen []string + require.NoError(t, driver.ListDir(nil, p, func(fi os.FileInfo) error { + seen = append(seen, fi.Name()) + return nil + })) + require.Equal(t, []string{"evil.dtd"}, seen) + require.Equal(t, []string{p}, inner.listed) + }) +} + +func TestFTPDownloadCorrelation(t *testing.T) { + newFTP := func(t *testing.T) (*FTPServer, *HTTPServer, string, string) { + t.Helper() + h, id, secret := uploadTestServer(t, true) + h.options.Token = "shared-token" + require.NoError(t, h.options.Storage.SetID(h.options.Token)) + + resp := doUpload(t, h, uploadBody(t, id, secret, map[string][]byte{"evil.dtd": []byte("payload")})) + require.Equal(t, http.StatusOK, resp.StatusCode) + + return &FTPServer{options: h.options}, h, id, secret + } + + t.Run("download attributed to the owning session", func(t *testing.T) { + ftp, h, id, _ := newFTP(t) + + hosted := "/" + uploadsDirName + "/" + id + "/evil.dtd" + ftp.recordInteractionForPath("198.51.100.4:3333", "RETR "+hosted+"\ndownloaded file", hosted) + + item, err := h.options.Storage.GetCacheItem(id) + require.NoError(t, err) + require.Len(t, item.Data, 1, "the owning session should see its own file being fetched") + + record := &Interaction{} + require.NoError(t, jsoniter.Unmarshal([]byte(item.Data[0]), record)) + require.Equal(t, "ftp", record.Protocol) + require.Equal(t, id, record.UniqueID) + + // And it must not also land in the shared bucket, or every client sees it. + shared, err := h.options.Storage.GetCacheItem(h.options.Token) + require.NoError(t, err) + require.Empty(t, shared.Data, "an attributed download must not be duplicated to the token bucket") + }) + + t.Run("unattributable interactions still use the token bucket", func(t *testing.T) { + ftp, h, _, _ := newFTP(t) + + // A login has no path, and a path outside any session cannot be attributed. + ftp.recordInteraction("198.51.100.4:3333", "USER anonymous\nlogging in") + ftp.recordInteractionForPath("198.51.100.4:3333", "RETR /nope\ndownloaded file", "/nope") + + shared, err := h.options.Storage.GetCacheItem(h.options.Token) + require.NoError(t, err) + require.Len(t, shared.Data, 2, "unattributable FTP noise keeps its existing behaviour") + }) + + t.Run("path for an unknown session is not attributed", func(t *testing.T) { + ftp, h, _, _ := newFTP(t) + unknown := "c6rj61aciaeutn2ae681" + + require.Equal(t, "", ftp.correlationIDFromPath("/"+uploadsDirName+"/"+unknown+"/evil.dtd"), + "only live sessions should be attributed") + + shared, err := h.options.Storage.GetCacheItem(h.options.Token) + require.NoError(t, err) + require.Empty(t, shared.Data) + }) + + // Paths are cleaned before the leading segment is taken, so traversal can + // only ever resolve to the session it actually points at -- never to a + // different one, and never outside the hosted-files root. + t.Run("traversal resolves before attribution", func(t *testing.T) { + ftp, _, id, _ := newFTP(t) + + // Leading ".." is dropped at the root, so this still names id. + require.Equal(t, id, ftp.correlationIDFromPath("/../"+uploadsDirName+"/"+id+"/evil.dtd")) + require.Equal(t, id, ftp.correlationIDFromPath("//"+uploadsDirName+"/"+id+"/evil.dtd")) + + // Climbing out of a session directory stops attributing to it. + require.Equal(t, "", ftp.correlationIDFromPath("/"+uploadsDirName+"/"+id+"/../other/evil.dtd")) + require.Equal(t, "", ftp.correlationIDFromPath("/"+uploadsDirName+"/"+id+"/..")) + + // A path outside the uploads directory is not ours, however much it looks + // like a session: an operator directory at the FTP root shares that shape. + require.Equal(t, "", ftp.correlationIDFromPath("/"+id+"/evil.dtd"), + "only paths under the uploads directory may be attributed") + }) +} From b0a3832c4b89da9e41e3eb96cba441b5b1ad6d84 Mon Sep 17 00:00:00 2001 From: nisay759 Date: Fri, 31 Jul 2026 00:15:27 +0200 Subject: [PATCH 08/20] client: add UploadFiles and server capability plumbing performRegistration now decodes the typed RegisterResponse and stores the advertised capabilities, so the client knows whether the server hosts files, and its limits, before trying. Capabilities live in an atomic.Value because the keep-alive goroutine re-registers periodically. The check on the message field is unchanged, so behaviour against an older server is identical and a missing capabilities block simply reads as "unknown". UploadFiles targets only the server the client registered with. A Client holds one correlation id, registered with whichever server answered first, so the rest of -s never saw it and would reject the upload. Posting to them anyway would be worse than useless: a server without -upload has no route for the request, so it falls through to the catch-all handler that records whole requests as interactions, and the file would end up stored there. For the same reason the client fails closed. A 501, 404 or 405 is reported as ErrUploadUnsupported rather than retried or ignored, and a server that has advertised no upload support is not contacted at all. Uploads refuse the plaintext HTTP fallback that registration is allowed to use, since the request carries both the file and the session secret key. Loopback is exempt so local testing still works. Files are validated locally first -- exists, regular, non-empty, within the advertised size and count limits, name acceptable to the server, no two paths sharing a basename -- so mistakes surface immediately with a clear message instead of as a 400. Co-Authored-By: Claude ErrUploadUnsupported is declared with errors.New rather than errkit.New: errkit compares errors by message, so errors.Is against an errkit sentinel matches anything whose message contains it, in either direction. A plain error keeps the comparison exact, which matters as soon as a more specific sentinel is built on top of this one. --- pkg/client/client.go | 28 ++++- pkg/client/upload.go | 227 +++++++++++++++++++++++++++++++++++++ pkg/client/upload_test.go | 231 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 480 insertions(+), 6 deletions(-) create mode 100644 pkg/client/upload.go create mode 100644 pkg/client/upload_test.go diff --git a/pkg/client/client.go b/pkg/client/client.go index bea9caa5..e5f35a53 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -67,6 +67,10 @@ type Client struct { token string correlationIdLength int CorrelationIdNonceLength int + // capabilities holds the *server.Capabilities advertised at registration. + // Written from performRegistration, which the keep-alive goroutine also + // calls, hence atomic.Value rather than a bare field. + capabilities atomic.Value } // Options contains configuration options for interactsh client @@ -632,16 +636,21 @@ func (c *Client) performRegistration(serverURL string, payload []byte) error { data, _ := io.ReadAll(resp.Body) return fmt.Errorf("could not register to server: %s", string(data)) } - response := make(map[string]interface{}) - if err := json.NewDecoder(resp.Body).Decode(&response); err != nil { + response := &server.RegisterResponse{} + if err := json.NewDecoder(resp.Body).Decode(response); err != nil { return errkit.Wrap(err, "could not register to server") } - message, ok := response["message"] - if !ok { + if response.Message == "" { return errors.New("could not get register response") } - if message.(string) != "registration successful" { - return fmt.Errorf("could not get register response: %s", message.(string)) + if response.Message != "registration successful" { + return fmt.Errorf("could not get register response: %s", response.Message) + } + + // Nil against a server predating capability advertisement, which callers + // treat as "unknown" rather than "unsupported". + if response.Capabilities != nil { + c.capabilities.Store(response.Capabilities) } c.State.Store(Idle) @@ -649,6 +658,13 @@ func (c *Client) performRegistration(serverURL string, payload []byte) error { return nil } +// Capabilities returns the optional features advertised by the server at +// registration, or nil if the server did not advertise any. +func (c *Client) Capabilities() *server.Capabilities { + caps, _ := c.capabilities.Load().(*server.Capabilities) + return caps +} + // URL returns a new URL that can be used for external interaction requests. func (c *Client) URL() string { if c.State.Load() == Closed { diff --git a/pkg/client/upload.go b/pkg/client/upload.go new file mode 100644 index 00000000..1b2b290e --- /dev/null +++ b/pkg/client/upload.go @@ -0,0 +1,227 @@ +package client + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "os" + "path/filepath" + "regexp" + "strings" + + "github.com/projectdiscovery/interactsh/pkg/server" + "github.com/projectdiscovery/retryablehttp-go" + "github.com/projectdiscovery/utils/errkit" +) + +// ErrUploadUnsupported is returned when the interactsh server does not offer +// file hosting, either because it was started without -upload or because it +// predates the feature. +// +// Declared with errors.New rather than errkit.New so that errors.Is stays exact: +// errkit compares errors by message, which makes a sentinel and anything whose +// message contains it match in both directions. +var ErrUploadUnsupported = errors.New("interactsh server does not support file upload") + +// defaultMaxUploadFileSize bounds a local file when the server has not told us +// its limit, so a mistyped -file cannot try to push a huge file over the wire. +const defaultMaxUploadFileSize = 1 << 20 + +// uploadNameRe mirrors the server's allowlist, so an unusable name is rejected +// locally with a clear message instead of as a 400 from the server. +var uploadNameRe = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$`) + +// UploadedFile describes a file hosted by the interactsh server. +type UploadedFile struct { + Name string `json:"name"` + Size int64 `json:"size"` + SHA256 string `json:"sha256"` + HTTPPath string `json:"http-path"` + FTPPath string `json:"ftp-path"` +} + +// UploadFiles uploads local files to the interactsh server this client +// registered with, to be hosted against its correlation ID. +// +// Only the registered server is targeted. A Client holds a single correlation +// ID, registered with whichever server answered first, so the other servers in +// -s have never seen it and would reject the upload. Posting blindly to them +// would also be actively harmful: a server without -upload has no route for the +// request, so it falls through to the catch-all handler that records whole +// requests as interactions, and the file would be stored there anyway. +func (c *Client) UploadFiles(paths []string) ([]UploadedFile, error) { + c.busy.RLock() + defer c.busy.RUnlock() + + if c.State.Load() == Closed { + return nil, errkit.New("client is closed") + } + if c.serverURL == nil { + return nil, errkit.New("client is not registered with any server") + } + + // Fail closed when the server has told us it cannot host files. + if caps := c.Capabilities(); caps != nil && !caps.Upload { + return nil, ErrUploadUnsupported + } + + // Uploads carry the file and the secret key, so they must never traverse + // the plaintext fallback that registration is permitted to use. + if c.serverURL.Scheme != "https" && !isLoopbackURL(c.serverURL.Host) { + return nil, errkit.New("refusing to upload over plaintext http to " + c.serverURL.Host + + "; use an https server url") + } + + request, err := c.buildUploadRequest(paths) + if err != nil { + return nil, err + } + + payload, err := json.Marshal(request) + if err != nil { + return nil, errkit.Wrap(err, "could not encode upload request") + } + + ctx := context.WithValue(context.Background(), retryablehttp.RETRY_MAX, 0) + req, err := retryablehttp.NewRequestWithContext(ctx, http.MethodPost, c.serverURL.String()+"/upload", bytes.NewReader(payload)) + if err != nil { + return nil, errkit.Wrap(err, "could not create upload request") + } + req.ContentLength = int64(len(payload)) + req.Header.Set("Content-Type", "application/json") + if c.token != "" { + req.Header.Add("Authorization", c.token) + } + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, errkit.Wrap(err, "could not make upload request") + } + defer func() { + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + }() + + switch resp.StatusCode { + case http.StatusOK: + case http.StatusNotImplemented, http.StatusNotFound, http.StatusMethodNotAllowed: + return nil, ErrUploadUnsupported + case http.StatusUnauthorized: + return nil, errkit.New("invalid token provided for interactsh server") + default: + body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + return nil, fmt.Errorf("could not upload files (%s): %s", resp.Status, strings.TrimSpace(string(body))) + } + + response := &server.UploadResponse{} + if err := json.NewDecoder(resp.Body).Decode(response); err != nil { + return nil, errkit.Wrap(err, "could not decode upload response") + } + + files := make([]UploadedFile, 0, len(response.Files)) + for _, f := range response.Files { + files = append(files, UploadedFile{ + Name: f.Name, + Size: f.Size, + SHA256: f.SHA256, + HTTPPath: f.HTTPPath, + FTPPath: f.FTPPath, + }) + } + return files, nil +} + +// buildUploadRequest reads and validates the local files before anything is +// sent, so a bad path or an oversize file fails immediately and clearly. +func (c *Client) buildUploadRequest(paths []string) (*server.UploadRequest, error) { + maxFileSize, maxFiles := int64(defaultMaxUploadFileSize), 0 + if caps := c.Capabilities(); caps != nil { + if caps.UploadMaxFileSize > 0 { + maxFileSize = caps.UploadMaxFileSize + } + maxFiles = caps.UploadMaxFiles + } + if maxFiles > 0 && len(paths) > maxFiles { + return nil, fmt.Errorf("%d files requested but the server accepts at most %d", len(paths), maxFiles) + } + + request := &server.UploadRequest{CorrelationID: c.correlationID, SecretKey: c.secretKey} + seen := make(map[string]string, len(paths)) + + for _, p := range paths { + info, err := os.Stat(p) + if err != nil { + return nil, errkit.Wrap(err, "could not read file "+p) + } + if !info.Mode().IsRegular() { + return nil, fmt.Errorf("%s is not a regular file", p) + } + if info.Size() == 0 { + return nil, fmt.Errorf("%s is empty", p) + } + if info.Size() > maxFileSize { + return nil, fmt.Errorf("%s is %d bytes, the server accepts at most %d", p, info.Size(), maxFileSize) + } + + name := filepath.Base(p) + if !uploadNameRe.MatchString(name) || name == "." || name == ".." { + return nil, fmt.Errorf("%s has a name the server will not accept; "+ + "use only letters, digits, dot, dash and underscore", p) + } + if previous, dup := seen[name]; dup { + return nil, fmt.Errorf("%s and %s would both be hosted as %q", previous, p, name) + } + seen[name] = p + + data, err := os.ReadFile(p) + if err != nil { + return nil, errkit.Wrap(err, "could not read file "+p) + } + request.Files = append(request.Files, server.UploadFileRequest{ + Name: name, + Data: base64.StdEncoding.EncodeToString(data), + }) + } + + if len(request.Files) == 0 { + return nil, errkit.New("no files to upload") + } + return request, nil +} + +// FileURL returns the URL a target should fetch to retrieve a hosted file. +// payloadHost is any host produced by URL(); only its correlation ID prefix is +// significant to the server, so a single call to URL() serves every file. +func (c *Client) FileURL(payloadHost string, file UploadedFile) string { + scheme := "https" + if c.serverURL != nil && c.serverURL.Scheme != "" { + scheme = c.serverURL.Scheme + } + return scheme + "://" + payloadHost + file.HTTPPath +} + +// FTPFileURL returns the ftp:// URL for a hosted file. FTP has no host-based +// routing, so the correlation ID travels in the path instead. +func (c *Client) FTPFileURL(payloadHost string, file UploadedFile) string { + return "ftp://" + payloadHost + file.FTPPath +} + +// isLoopbackURL reports whether a host refers to the local machine, where a +// plaintext upload is not exposed to the network. +func isLoopbackURL(host string) bool { + name := host + if h, _, err := net.SplitHostPort(host); err == nil { + name = h + } + switch strings.ToLower(name) { + case "localhost", "127.0.0.1", "::1", "[::1]": + return true + } + return false +} diff --git a/pkg/client/upload_test.go b/pkg/client/upload_test.go new file mode 100644 index 00000000..d2f88b12 --- /dev/null +++ b/pkg/client/upload_test.go @@ -0,0 +1,231 @@ +package client + +import ( + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "sync/atomic" + "testing" + + "github.com/projectdiscovery/interactsh/pkg/server" + "github.com/projectdiscovery/retryablehttp-go" + "github.com/stretchr/testify/require" +) + +// newUploadClient returns a client pointed at handler, already "registered". +func newUploadClient(t *testing.T, handler http.HandlerFunc, caps *server.Capabilities) *Client { + t.Helper() + + ts := httptest.NewServer(handler) + t.Cleanup(ts.Close) + + parsed, err := url.Parse(ts.URL) + require.NoError(t, err) + + c := &Client{ + correlationID: "c6rj61aciaeutn2ae680", + secretKey: "6a1b0e5c-3f2d-4a7b-8c9d-0e1f2a3b4c5d", + serverURL: parsed, + httpClient: retryablehttp.NewClient(retryablehttp.DefaultOptionsSingle), + correlationIdLength: 20, + CorrelationIdNonceLength: 13, + } + c.State.Store(Idle) + if caps != nil { + c.capabilities.Store(caps) + } + return c +} + +func writeTempFile(t *testing.T, name string, content []byte) string { + t.Helper() + p := filepath.Join(t.TempDir(), name) + require.NoError(t, os.WriteFile(p, content, 0o600)) + return p +} + +func TestUploadFiles(t *testing.T) { + caps := &server.Capabilities{Upload: true, UploadMaxFileSize: 1024, UploadMaxFiles: 5, FTP: true} + + t.Run("posts the expected request and decodes the response", func(t *testing.T) { + var ( + gotAuth string + gotPath string + gotMethod string + gotBody server.UploadRequest + ) + c := newUploadClient(t, func(w http.ResponseWriter, r *http.Request) { + gotAuth, gotPath, gotMethod = r.Header.Get("Authorization"), r.URL.Path, r.Method + require.NoError(t, json.NewDecoder(r.Body).Decode(&gotBody)) + + _ = json.NewEncoder(w).Encode(&server.UploadResponse{ + Message: "upload successful", + Files: []server.UploadedFileResponse{{ + Name: "evil.dtd", Size: 7, SHA256: "abc", + HTTPPath: "/f/evil.dtd", FTPPath: "/.interactsh-user-uploads/c6rj61aciaeutn2ae680/evil.dtd", + }}, + }) + }, caps) + c.token = "sekrit" + + files, err := c.UploadFiles([]string{writeTempFile(t, "evil.dtd", []byte("payload"))}) + require.NoError(t, err) + + require.Equal(t, http.MethodPost, gotMethod) + require.Equal(t, "/upload", gotPath) + require.Equal(t, "sekrit", gotAuth) + require.Equal(t, "c6rj61aciaeutn2ae680", gotBody.CorrelationID) + require.Equal(t, c.secretKey, gotBody.SecretKey) + require.Len(t, gotBody.Files, 1) + require.Equal(t, "evil.dtd", gotBody.Files[0].Name) + require.Equal(t, "cGF5bG9hZA==", gotBody.Files[0].Data, "content should be base64 encoded") + + require.Len(t, files, 1) + require.Equal(t, "/f/evil.dtd", files[0].HTTPPath) + }) + + t.Run("501 reports unsupported", func(t *testing.T) { + c := newUploadClient(t, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotImplemented) + }, nil) + + _, err := c.UploadFiles([]string{writeTempFile(t, "a.dtd", []byte("x"))}) + require.ErrorIs(t, err, ErrUploadUnsupported) + }) + + t.Run("404 and 405 report unsupported", func(t *testing.T) { + for _, code := range []int{http.StatusNotFound, http.StatusMethodNotAllowed} { + c := newUploadClient(t, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(code) + }, nil) + _, err := c.UploadFiles([]string{writeTempFile(t, "a.dtd", []byte("x"))}) + require.ErrorIs(t, err, ErrUploadUnsupported, "status %d", code) + } + }) + + t.Run("server error message is surfaced", func(t *testing.T) { + c := newUploadClient(t, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusRequestEntityTooLarge) + _, _ = w.Write([]byte(`{"error":"file is too big"}`)) + }, caps) + + _, err := c.UploadFiles([]string{writeTempFile(t, "a.dtd", []byte("x"))}) + require.Error(t, err) + require.Contains(t, err.Error(), "file is too big") + }) + + t.Run("advertised absence of upload fails without a request", func(t *testing.T) { + var called atomic.Bool + c := newUploadClient(t, func(w http.ResponseWriter, r *http.Request) { + called.Store(true) + }, &server.Capabilities{Upload: false}) + + _, err := c.UploadFiles([]string{writeTempFile(t, "a.dtd", []byte("x"))}) + require.ErrorIs(t, err, ErrUploadUnsupported) + require.False(t, called.Load(), "must not contact a server known not to support uploads") + }) + + t.Run("local validation happens before any request", func(t *testing.T) { + cases := []struct { + name string + paths func(t *testing.T) []string + want string + }{ + {"missing file", func(t *testing.T) []string { + return []string{filepath.Join(t.TempDir(), "absent.dtd")} + }, "could not read file"}, + {"directory", func(t *testing.T) []string { + return []string{t.TempDir()} + }, "not a regular file"}, + {"empty file", func(t *testing.T) []string { + return []string{writeTempFile(t, "empty.dtd", nil)} + }, "is empty"}, + {"oversize", func(t *testing.T) []string { + return []string{writeTempFile(t, "big.dtd", make([]byte, 4096))} + }, "accepts at most"}, + {"unusable name", func(t *testing.T) []string { + return []string{writeTempFile(t, ".hidden", []byte("x"))} + }, "will not accept"}, + {"too many files", func(t *testing.T) []string { + var paths []string + for i := 0; i < 6; i++ { + paths = append(paths, writeTempFile(t, "f.dtd", []byte("x"))) + } + return paths + }, "accepts at most"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var called atomic.Bool + c := newUploadClient(t, func(w http.ResponseWriter, r *http.Request) { + called.Store(true) + }, caps) + + _, err := c.UploadFiles(tc.paths(t)) + require.Error(t, err) + require.Contains(t, err.Error(), tc.want) + require.False(t, called.Load(), "must fail before contacting the server") + }) + } + }) + + t.Run("duplicate basenames are rejected", func(t *testing.T) { + c := newUploadClient(t, func(w http.ResponseWriter, r *http.Request) {}, caps) + + a := writeTempFile(t, "evil.dtd", []byte("one")) + b := writeTempFile(t, "evil.dtd", []byte("two")) + _, err := c.UploadFiles([]string{a, b}) + require.Error(t, err) + require.Contains(t, err.Error(), "would both be hosted as") + }) + + t.Run("refuses plaintext http to a remote server", func(t *testing.T) { + c := newUploadClient(t, func(w http.ResponseWriter, r *http.Request) {}, caps) + c.serverURL = &url.URL{Scheme: "http", Host: "oast.example.com"} + + _, err := c.UploadFiles([]string{writeTempFile(t, "a.dtd", []byte("x"))}) + require.Error(t, err) + require.Contains(t, err.Error(), "refusing to upload over plaintext http", + "the file and the secret key must not go over the wire in clear") + }) + + t.Run("closed client", func(t *testing.T) { + c := newUploadClient(t, func(w http.ResponseWriter, r *http.Request) {}, caps) + c.State.Store(Closed) + + _, err := c.UploadFiles([]string{writeTempFile(t, "a.dtd", []byte("x"))}) + require.Error(t, err) + require.False(t, errors.Is(err, ErrUploadUnsupported)) + }) +} + +func TestFileURLComposition(t *testing.T) { + file := UploadedFile{Name: "evil.dtd", HTTPPath: "/f/evil.dtd", FTPPath: "/.interactsh-user-uploads/c6rj61aciaeutn2ae680/evil.dtd"} + // 20-char correlation id plus a 13-char nonce, as URL() composes it. + host := "c6rj61aciaeutn2ae680xk4tqy8pqhwmi.oast.test" + + t.Run("https server", func(t *testing.T) { + c := &Client{serverURL: &url.URL{Scheme: "https", Host: "oast.test"}} + require.Equal(t, "https://"+host+"/f/evil.dtd", c.FileURL(host, file)) + require.Equal(t, "ftp://"+host+"/.interactsh-user-uploads/c6rj61aciaeutn2ae680/evil.dtd", c.FTPFileURL(host, file)) + }) + + t.Run("http server", func(t *testing.T) { + c := &Client{serverURL: &url.URL{Scheme: "http", Host: "127.0.0.1:8080"}} + require.Equal(t, "http://"+host+"/f/evil.dtd", c.FileURL(host, file)) + }) +} + +func TestIsLoopbackURL(t *testing.T) { + for _, host := range []string{"localhost", "127.0.0.1", "127.0.0.1:8080", "localhost:8080", "[::1]:8080"} { + require.True(t, isLoopbackURL(host), "expected %q to be loopback", host) + } + for _, host := range []string{"oast.test", "example.com:8080", "10.0.0.1"} { + require.False(t, isLoopbackURL(host), "expected %q not to be loopback", host) + } +} From a3edfe9df7dde62a788f17a27525a736f215f7c2 Mon Sep 17 00:00:00 2001 From: nisay759 Date: Fri, 31 Jul 2026 00:16:58 +0200 Subject: [PATCH 09/20] client: add -file flag to host payload files interactsh-client -file evil.dtd uploads the file to the registered server and prints the URL a target should fetch, alongside the usual payload URLs. The flag uses goflags.StringSliceOptions rather than the FileCommaSeparatedStringSliceOptions used by -match and -filter: that variant reads the named file and splits its contents, which here would turn a DTD into a list of filenames. Short name is -fl because -f is filter. All files share a single payload host, so the target performs one DNS lookup and the output stays consistent; only the correlation id prefix is significant to the server, so any nonce works. The ftp:// URL is printed only when the server advertises an FTP listener, since otherwise it would never connect. Upload failure is fatal. The user asked to host a payload, and continuing without it yields a confusing run where no interaction ever arrives; a server without -upload gets a message naming the flag it needs. Co-Authored-By: Claude --- cmd/interactsh-client/main.go | 58 ++++++++++++++++++++++++++++++++++- pkg/options/client_options.go | 1 + 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/cmd/interactsh-client/main.go b/cmd/interactsh-client/main.go index d0556728..9eb1cbf1 100644 --- a/cmd/interactsh-client/main.go +++ b/cmd/interactsh-client/main.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "os" "os/signal" @@ -44,6 +45,11 @@ func main() { flagSet.CreateGroup("input", "Input", flagSet.StringVarP(&cliOptions.ServerURL, "server", "s", defaultOpts.ServerURL, "interactsh server(s) to use"), + // StringSliceOptions, not the FileCommaSeparated variant used by + // -match/-filter: that one reads the file and splits its contents, + // which for -file would turn a DTD into a list of names. + flagSet.StringSliceVarP(&cliOptions.Files, "file", "fl", nil, + "local file(s) to upload and host on the interactsh server", goflags.StringSliceOptions), ) flagSet.CreateGroup("config", "config", @@ -171,6 +177,10 @@ func main() { gologger.Fatal().Msgf("Could not create client: %s\n", err) } + // Uploads must follow registration, since the server verifies the session, + // and precede the payload listing so every URL is shown together. + fileURLs := uploadFiles(cliOptions.Files, client) + interactshURLs := generatePayloadURL(cliOptions.NumberOfPayloads, client) gologger.Info().Msgf("Listing %d payload for OOB Testing\n", cliOptions.NumberOfPayloads) @@ -180,8 +190,16 @@ func main() { warnIfServerLacksIPv6(client) + if len(fileURLs) > 0 { + gologger.Info().Msgf("Hosting %d file(s) for OOB Testing\n", len(cliOptions.Files)) + for _, fileURL := range fileURLs { + gologger.Info().Msgf("%s\n", fileURL) + } + } + if cliOptions.StorePayload && cliOptions.StorePayloadFile != "" { - if err := os.WriteFile(cliOptions.StorePayloadFile, []byte(strings.Join(interactshURLs, "\n")), 0644); err != nil { + stored := append(append([]string{}, interactshURLs...), fileURLs...) + if err := os.WriteFile(cliOptions.StorePayloadFile, []byte(strings.Join(stored, "\n")), 0644); err != nil { gologger.Fatal().Msgf("Could not write to payload output file: %s\n", err) } } @@ -320,6 +338,44 @@ func generatePayloadURL(numberOfPayloads int, client *client.Client) []string { return interactshURLs } +// uploadFiles hosts local files on the interactsh server and returns the URLs a +// target should fetch. It returns nil when no files were requested. +func uploadFiles(paths []string, c *client.Client) []string { + if len(paths) == 0 { + return nil + } + + uploaded, err := c.UploadFiles(paths) + if err != nil { + if errors.Is(err, client.ErrUploadUnsupported) { + gologger.Fatal().Msgf("Server does not accept file uploads; it must be started with -upload\n") + } + // Fatal rather than a warning: the user asked to host a payload, and + // carrying on without it produces a confusing "no interaction" result. + gologger.Fatal().Msgf("Could not upload files: %s\n", err) + } + + // One payload host for every file, so the target performs a single DNS + // lookup and the output is consistent. Any nonce works, since the server + // only reads the correlation id prefix. + host := c.URL() + withFTP := false + if caps := c.Capabilities(); caps != nil { + withFTP = caps.FTP + } + + var urls []string + for _, file := range uploaded { + urls = append(urls, c.FileURL(host, file)) + // Only when the server actually runs an FTP listener, otherwise the + // URL would never connect. + if withFTP { + urls = append(urls, c.FTPFileURL(host, file)) + } + } + return urls +} + func writeOutput(outputFile *os.File, builder *bytes.Buffer) { if outputFile != nil { _, _ = outputFile.Write(builder.Bytes()) diff --git a/pkg/options/client_options.go b/pkg/options/client_options.go index 835380ec..c6e8c804 100644 --- a/pkg/options/client_options.go +++ b/pkg/options/client_options.go @@ -9,6 +9,7 @@ import ( type CLIClientOptions struct { Match goflags.StringSlice Filter goflags.StringSlice + Files goflags.StringSlice Config string Version bool ServerURL string From a7caaeeff7fe0ea220de7a91ca170cc43b2148d0 Mon Sep 17 00:00:00 2001 From: nisay759 Date: Fri, 31 Jul 2026 00:56:42 +0200 Subject: [PATCH 10/20] Immediate upload cleanup on deregister, fix ftp URL port, update README Three fixes found by running the feature end to end. deregisterHandler now deletes the session's files synchronously. The cache eviction hook reached through RemoveID only enqueues the directory, leaving a window in which a client that had just deregistered could still fetch its own hosted files. Blocking is safe in the handler -- unlike the cache event goroutine, which is why the queue exists at all -- and the deletion is idempotent, so the queued removal that follows is a no-op. TestDeregisterRemovesFilesSynchronously pins the ordering: the fixture never starts the deleter goroutine, so a queued-only removal leaves the directory in place and fails the assertion. FTPFileURL carried the port from the payload host, which is the HTTP listener's port and says nothing about where FTP is bound -- against a server on :8080 the client printed ftp://host:8080/... which cannot connect. Any port is now dropped so the URL uses the FTP default. README gains a Client File Hosting section covering the flags, the URL shapes and the cleanup behaviour, and stating plainly that hosted files are readable by anyone who learns the correlation id. That id is deliberately leaked to the target, so it appears in the target's DNS logs and in passive DNS; a target can fetch the payload to fingerprint the tester, and that fetch shows up as an interaction. The self-hosted-only warning and the tmpfs caveat for the default upload directory are documented alongside. Verified end to end against a real server: upload, HTTP fetch with matching bytes and hardened headers, FTP fetch with matching bytes, empty FTP root listing, an interaction recorded for the fetch with the body elided and no file content reaching the client, and the session directory removed on deregistration. Co-Authored-By: Claude --- README.md | 119 ++++++++++++++++++++++++++++++ pkg/client/upload.go | 10 ++- pkg/client/upload_test.go | 8 ++ pkg/server/ftp_upload_test.go | 4 +- pkg/server/http_server.go | 10 +++ pkg/server/upload_handler_test.go | 29 ++++++++ pkg/server/upload_serve_test.go | 4 +- 7 files changed, 179 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index e67e4cf7..881d31e5 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,7 @@ - NTLM/SMB/FTP(S)/RESPONDER Listener **(self-hosted)** - Wildcard / Protected Interactions **(self-hosted)** - Customizable Index / File hosting **(self-hosted)** +- Client file hosting for second-stage OOB payloads **(self-hosted)** - Customizable Payload Length **(self-hosted)** - Custom SSL Certificate **(self-hosted)** @@ -654,6 +655,124 @@ interactsh-server -d hackwithautomation.com -http-directory ./paylods ![image](https://user-images.githubusercontent.com/8293321/179396480-d5ff8399-8b91-48aa-b21f-c67e40e80945.png) +## Client File Hosting + +Where `-http-directory` hosts operator-supplied files globally, `-upload` lets a **client** host files +against its own correlation ID. This is aimed at second-stage out-of-band vulnerabilities — XXE with an +external DTD, XSLT includes, JNDI staging — where the target must fetch a payload file before the +callback fires. Each fetch is recorded as an interaction, so the second stage is visible in the client +output. + +> [!WARNING] +> `-upload` is intended for **self-hosted servers only**. Enabling it on a public instance turns it into +> anonymous file hosting on a domain with a valid wildcard certificate, which is a magnet for malware +> staging, and blocklists act on the registrable domain — one abusive sample affects every user of that +> domain. It is off by default and implies authentication when enabled. + +Start a server with uploads enabled: + +```bash +interactsh-server -d hackwithautomation.com -upload -ftp +``` + +Then point a client at it with one or more files: + +```bash +interactsh-client -s https://hackwithautomation.com -t -file evil.dtd +``` + +```console +[INF] Listing 1 payload for OOB Testing +[INF] c6rj61aciaeutn2ae680cndmnioyyyyyn.hackwithautomation.com +[INF] Hosting 1 file(s) for OOB Testing +[INF] https://c6rj61aciaeutn2ae680xk4tqy8pqhwmi.hackwithautomation.com/f/evil.dtd +[INF] ftp://c6rj61aciaeutn2ae680xk4tqy8pqhwmi.hackwithautomation.com/c6rj61aciaeutn2ae680/evil.dtd +``` + +Files are served over HTTP(S), and over FTP(S) as well when `-ftp` is enabled. Responses are always +`Content-Type: application/octet-stream` with `Content-Disposition: attachment`, so the server never +renders client-supplied HTML or SVG on its own domain; DTD, XSLT and JNDI consumers ignore content type, +so this costs nothing for the intended use. + +When the target fetches the file, the fetch arrives in the client like any other interaction — which is +the point: it is the evidence that the first stage of the payload actually executed. The response body +is replaced by a digest so a large payload is not copied back into the interaction stream on every +fetch: + +```console +[c6rj61aciaeutn2ae680xk4tqy8pqhwmi] Received HTTP interaction from 203.0.113.7 at 2026-08-05 15:47:19 +------------ +HTTP Request +------------ + +GET /f/evil.dtd HTTP/1.1 +Host: c6rj61aciaeutn2ae680xk4tqy8pqhwmi.hackwithautomation.com +Accept: */* +User-Agent: curl/8.18.0 + +------------- +HTTP Response +------------- + +HTTP/1.1 200 OK +Content-Type: application/octet-stream +Content-Disposition: attachment; filename="evil.dtd" +Content-Length: 144 + +[body elided: 144 bytes of uploaded file "evil.dtd", sha256 0c1b960b076cdff8666f1f302dddd8f3ff0e6ed4b6c09002fbe6d1cdbb5d68f8] +``` + +Whatever second-stage callback the payload then triggers arrives as a further interaction on the same +correlation ID, so both stages land in one client. + +A client asked to host files against a server that was not started with `-upload` says so and stops, +rather than silently continuing without the payload: + +```console +$ interactsh-client -s https://hackwithautomation.com -t -file evil.dtd +[FTL] Server does not accept file uploads; it must be started with -upload +``` + +Server-side options: + +| Flag | Default | Description | +| --- | --- | --- | +| `-upload` | off | enable client file upload and hosting | +| `-ud, -upload-directory` | temporary dir | where uploaded files are stored | +| `-umfs, -upload-max-file-size` | `1mb` | maximum size of a single file | +| `-umf, -upload-max-files` | `5` | maximum files per session | +| `-umts, -upload-max-total-size` | `1gb` | maximum total bytes across all sessions | +| `-ut, -upload-ttl` | `24h` | maximum lifetime of uploaded files | + +Files are removed when the client deregisters, when its session leaves the cache, and in any case once +`-upload-ttl` has elapsed since the last upload for that session. + +Things worth knowing before enabling it: + +- **Hosted files are readable by anyone who learns the correlation ID.** That ID is deliberately leaked + to the target — it appears in every DNS query the target's resolver makes, and so in its DNS logs, its + WAF, and passive-DNS aggregators. A target can fetch your payload to fingerprint your tooling, and + that fetch will appear in your interaction stream. Do not upload anything you would mind a target + reading. +- Uploads are authenticated with the session's correlation ID and secret key, so only the client that + owns a session can attach files to it. +- The client uploads to the **one server it registered with**. If `-s` lists several, files are hosted + only on the elected one; its payload URLs are the ones printed. **Pass a single server with `-file`:** + the client picks one at random from `-s` and cannot take upload support into account, since it only + learns that after registering — so a list mixing upload and non-upload servers fails at random. +- `-upload` cannot be combined with `-redis-url`. Hosted bytes are written to a single instance's local + filesystem, so with a storage backend shared between instances the other instances would advertise + files they do not have. The server refuses to start on that combination: + + ```console + $ interactsh-server -d hackwithautomation.com -upload -redis-url redis://127.0.0.1:6379/0 + [FTL] -upload cannot be used with -redis-url: hosted files are stored on a single instance's local filesystem + ``` +- Uploads refuse to travel over plaintext HTTP to a remote server, since the request carries both the + file and the session secret key. Use an `https://` server URL. +- The default upload directory is a temporary directory, which on many Linux distributions is + memory-backed. Set `-upload-directory` explicitly on a real deployment. + ## Dynamic HTTP Response Interactsh http server optionally enables responding with dynamic HTTP response by using query parameters. This feature can be enabled by using `-dr` or `-dynamic-resp` flag. diff --git a/pkg/client/upload.go b/pkg/client/upload.go index 1b2b290e..83c39345 100644 --- a/pkg/client/upload.go +++ b/pkg/client/upload.go @@ -208,8 +208,16 @@ func (c *Client) FileURL(payloadHost string, file UploadedFile) string { // FTPFileURL returns the ftp:// URL for a hosted file. FTP has no host-based // routing, so the correlation ID travels in the path instead. +// +// Any port on payloadHost is dropped: it belongs to the server's HTTP listener, +// which says nothing about where the FTP listener is bound, so carrying it over +// would produce a URL that cannot connect. func (c *Client) FTPFileURL(payloadHost string, file UploadedFile) string { - return "ftp://" + payloadHost + file.FTPPath + host := payloadHost + if h, _, err := net.SplitHostPort(host); err == nil { + host = h + } + return "ftp://" + host + file.FTPPath } // isLoopbackURL reports whether a host refers to the local machine, where a diff --git a/pkg/client/upload_test.go b/pkg/client/upload_test.go index d2f88b12..a6100d83 100644 --- a/pkg/client/upload_test.go +++ b/pkg/client/upload_test.go @@ -219,6 +219,14 @@ func TestFileURLComposition(t *testing.T) { c := &Client{serverURL: &url.URL{Scheme: "http", Host: "127.0.0.1:8080"}} require.Equal(t, "http://"+host+"/f/evil.dtd", c.FileURL(host, file)) }) + + // The payload host carries the HTTP listener's port, which tells us nothing + // about where FTP is bound, so it must not leak into the ftp:// URL. + t.Run("ftp url drops the http port", func(t *testing.T) { + c := &Client{serverURL: &url.URL{Scheme: "http", Host: "127.0.0.1:8080"}} + require.Equal(t, "ftp://"+host+"/.interactsh-user-uploads/c6rj61aciaeutn2ae680/evil.dtd", + c.FTPFileURL(host+":8080", file)) + }) } func TestIsLoopbackURL(t *testing.T) { diff --git a/pkg/server/ftp_upload_test.go b/pkg/server/ftp_upload_test.go index 31dd0862..59c563c3 100644 --- a/pkg/server/ftp_upload_test.go +++ b/pkg/server/ftp_upload_test.go @@ -1,12 +1,12 @@ package server import ( + "encoding/json" "net/http" "os" "testing" "time" - jsoniter "github.com/json-iterator/go" "github.com/stretchr/testify/require" ftpserver "goftp.io/server/v2" ) @@ -145,7 +145,7 @@ func TestFTPDownloadCorrelation(t *testing.T) { require.Len(t, item.Data, 1, "the owning session should see its own file being fetched") record := &Interaction{} - require.NoError(t, jsoniter.Unmarshal([]byte(item.Data[0]), record)) + require.NoError(t, json.Unmarshal([]byte(item.Data[0]), record)) require.Equal(t, "ftp", record.Protocol) require.Equal(t, id, record.UniqueID) diff --git a/pkg/server/http_server.go b/pkg/server/http_server.go index d55f809d..626fd80e 100644 --- a/pkg/server/http_server.go +++ b/pkg/server/http_server.go @@ -455,6 +455,16 @@ func (h *HTTPServer) deregisterHandler(w http.ResponseWriter, req *http.Request) return } + // Deleted synchronously rather than queued: the cache eviction hook fired + // by RemoveID above only enqueues the directory, leaving a window in which + // a client that has just deregistered could still fetch its own hosted + // files. Blocking here is safe -- unlike the cache's event goroutine, this + // handler can afford the filesystem call -- and the deletion is idempotent, + // so the queued removal that follows is a no-op. + if h.options.UploadStore != nil { + h.options.UploadStore.removeSessionNow(r.CorrelationID) + } + if h.options.RootTLD { for _, domain := range h.options.Domains { _ = h.options.Storage.RemoveConsumer(domain, r.CorrelationID) diff --git a/pkg/server/upload_handler_test.go b/pkg/server/upload_handler_test.go index 98a36547..ad9ebd7c 100644 --- a/pkg/server/upload_handler_test.go +++ b/pkg/server/upload_handler_test.go @@ -290,3 +290,32 @@ func TestRegisterAdvertisesCapabilities(t *testing.T) { require.False(t, out.Capabilities.Upload) }) } + +// TestDeregisterRemovesFilesSynchronously pins the ordering guarantee that a +// client which has just deregistered can no longer fetch its hosted files. +// +// The eviction hook reached through RemoveID only queues the directory, so a +// queued-only deregistration would leave the files readable until a background +// goroutine got to them. The fixture deliberately never starts that goroutine, +// which is what makes the distinction observable here. +func TestDeregisterRemovesFilesSynchronously(t *testing.T) { + h, id, secret := uploadTestServer(t, true) + + resp := doUpload(t, h, uploadBody(t, id, secret, map[string][]byte{"evil.dtd": []byte("payload")})) + require.Equal(t, http.StatusOK, resp.StatusCode) + + dir := h.options.UploadStore.sessionDir(id) + require.DirExists(t, dir, "upload should have created the session directory") + + body := fmt.Sprintf(`{"secret-key":%q,"correlation-id":%q}`, secret, id) + w := httptest.NewRecorder() + h.deregisterHandler(w, httptest.NewRequest(http.MethodPost, "http://example.com/deregister", strings.NewReader(body))) + require.Equal(t, http.StatusOK, w.Result().StatusCode) + + // No sleep and no polling: the files must be gone by the time the handler + // has returned, not merely scheduled for removal. + require.NoDirExists(t, dir, "deregistration must remove hosted files before returning") + + _, _, err := h.options.UploadStore.Open(id, "evil.dtd") + require.Error(t, err, "file must not be servable after deregistration") +} diff --git a/pkg/server/upload_serve_test.go b/pkg/server/upload_serve_test.go index 1eda44a1..f263f4af 100644 --- a/pkg/server/upload_serve_test.go +++ b/pkg/server/upload_serve_test.go @@ -1,6 +1,7 @@ package server import ( + "encoding/json" "fmt" "io" "net" @@ -9,7 +10,6 @@ import ( "strings" "testing" - jsoniter "github.com/json-iterator/go" "github.com/projectdiscovery/interactsh/pkg/storage" stringsutil "github.com/projectdiscovery/utils/strings" "github.com/stretchr/testify/require" @@ -137,7 +137,7 @@ func TestServeUploadedFileRecordsInteraction(t *testing.T) { require.Len(t, item.Data, 1, "the fetch must be visible to the client that owns the session") record := &Interaction{} - require.NoError(t, jsoniter.Unmarshal([]byte(item.Data[0]), record)) + require.NoError(t, json.Unmarshal([]byte(item.Data[0]), record)) require.Equal(t, "http", record.Protocol) require.Contains(t, record.RawRequest, "GET /f/evil.dtd") From d01e35a2ccf4585364a45aee883fae633b80c5c4 Mon Sep 17 00:00:00 2001 From: nisay759 Date: Tue, 4 Aug 2026 12:23:00 +0200 Subject: [PATCH 11/20] client: release the session when an upload aborts startup Upload support is only advertised in the register response, so -file has to register before it can discover the server does not accept uploads. The failure paths then called gologger.Fatal(), which exits without unwinding, leaving a registered session behind on the server until the eviction TTL reclaimed it. Observed end to end: three consecutive `-file` runs against a server started without -upload drove the reported session count to 1, 2, 3 while nothing was actually connected. Wind the session down the same way the signal handler does instead of leaving it stranded: persist it when -session-file was requested, otherwise deregister. The previous behaviour was the worst of both for -session-file users, since the session was neither released nor written anywhere they could resume it from. Verified against a live server: the session count now stays flat across repeated failures, -session-file writes a resumable session and deliberately keeps it registered, and a successful upload run is unchanged. Co-Authored-By: Claude --- cmd/interactsh-client/main.go | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/cmd/interactsh-client/main.go b/cmd/interactsh-client/main.go index 9eb1cbf1..34b6cf38 100644 --- a/cmd/interactsh-client/main.go +++ b/cmd/interactsh-client/main.go @@ -179,7 +179,7 @@ func main() { // Uploads must follow registration, since the server verifies the session, // and precede the payload listing so every URL is shown together. - fileURLs := uploadFiles(cliOptions.Files, client) + fileURLs := uploadFiles(cliOptions.Files, client, cliOptions.SessionFile) interactshURLs := generatePayloadURL(cliOptions.NumberOfPayloads, client) @@ -340,13 +340,28 @@ func generatePayloadURL(numberOfPayloads int, client *client.Client) []string { // uploadFiles hosts local files on the interactsh server and returns the URLs a // target should fetch. It returns nil when no files were requested. -func uploadFiles(paths []string, c *client.Client) []string { +// +// sessionFile mirrors -session-file: when set, the session is meant to outlive +// this process, so the failure paths below must not deregister it. +func uploadFiles(paths []string, c *client.Client, sessionFile string) []string { if len(paths) == 0 { return nil } uploaded, err := c.UploadFiles(paths) if err != nil { + // Registration already happened -- it has to, since upload support is + // only advertised in the register response -- so a session exists on the + // server. Wind it down the same way the signal handler does, rather than + // leaving it to sit until the eviction TTL: persist it if the user asked + // for a resumable session, otherwise deregister it. Doing neither would + // overstate the server's live session count for every client that trips + // this path, and would strand a session the user cannot resume. + if sessionFile != "" { + _ = c.SaveSessionTo(sessionFile) + } else { + _ = c.Close() + } if errors.Is(err, client.ErrUploadUnsupported) { gologger.Fatal().Msgf("Server does not accept file uploads; it must be started with -upload\n") } From 9dc056c29e048d81ec3d3f61d1342c6122621ea9 Mon Sep 17 00:00:00 2001 From: nisay759 Date: Wed, 5 Aug 2026 17:58:06 +0200 Subject: [PATCH 12/20] client: name the server when an upload is refused The client registers with one server out of -s, so "Server does not accept file uploads" left the reader guessing which one when several were listed. Both upload failure paths now name the elected server. Election makes this worse than it looks: parseServerURLs walks -s under sliceutil.VisitRandom and keeps the first server that registers, and upload support cannot influence that choice because it is only advertised in the registration response. A list mixing upload and non-upload servers therefore succeeds or fails at random, at roughly 1/N per run, regardless of the order the servers were written in. Measured over 20 runs against two local servers where only one had -upload: 6/4 with the capable server last, 4/6 with it first. So when -s named more than one server the message also says how this one was chosen and what to do about it, rather than reading as "none of my servers support uploads" on the runs that happen to elect one that does not: Server http://127.0.0.1:8080 does not accept file uploads; it must be started with -upload (chosen at random from the 2 servers in -s, so this may differ between runs; pass a single server with -file) uploadFiles now takes the CLI options rather than three positional arguments, two of them adjacent strings that were easy to transpose. Documented in the README alongside the existing single-server note. Co-Authored-By: Claude --- README.md | 10 ++++++- cmd/interactsh-client/main.go | 42 ++++++++++++++++++++++-------- cmd/interactsh-client/main_test.go | 36 +++++++++++++++++++++++++ 3 files changed, 76 insertions(+), 12 deletions(-) create mode 100644 cmd/interactsh-client/main_test.go diff --git a/README.md b/README.md index 881d31e5..b5c452a7 100644 --- a/README.md +++ b/README.md @@ -730,7 +730,15 @@ rather than silently continuing without the payload: ```console $ interactsh-client -s https://hackwithautomation.com -t -file evil.dtd -[FTL] Server does not accept file uploads; it must be started with -upload +[FTL] Server https://hackwithautomation.com does not accept file uploads; it must be started with -upload +``` + +The failing server is named because the client registers with only one of the servers in `-s`. When +several are listed, it also says how the choice was made, since the outcome can differ between runs: + +```console +$ interactsh-client -s https://a.example,https://b.example -t -file evil.dtd +[FTL] Server https://a.example does not accept file uploads; it must be started with -upload (chosen at random from the 2 servers in -s, so this may differ between runs; pass a single server with -file) ``` Server-side options: diff --git a/cmd/interactsh-client/main.go b/cmd/interactsh-client/main.go index 34b6cf38..0518a9d2 100644 --- a/cmd/interactsh-client/main.go +++ b/cmd/interactsh-client/main.go @@ -179,7 +179,7 @@ func main() { // Uploads must follow registration, since the server verifies the session, // and precede the payload listing so every URL is shown together. - fileURLs := uploadFiles(cliOptions.Files, client, cliOptions.SessionFile) + fileURLs := uploadFiles(client, cliOptions) interactshURLs := generatePayloadURL(cliOptions.NumberOfPayloads, client) @@ -338,17 +338,33 @@ func generatePayloadURL(numberOfPayloads int, client *client.Client) []string { return interactshURLs } +// electionHint explains which server the complaint is about when -s named more +// than one. The client registers with a single server chosen at random, and +// upload support cannot influence that choice because it is only advertised in +// the registration response. Without this, a mixed list reads as "none of my +// servers support uploads" on the runs that happen to elect one that does not. +func electionHint(serverList string) string { + var listed int + for _, s := range strings.Split(serverList, ",") { + if strings.TrimSpace(s) != "" { + listed++ + } + } + if listed < 2 { + return "" + } + return fmt.Sprintf(" (chosen at random from the %d servers in -s, so this may differ between runs;"+ + " pass a single server with -file)", listed) +} + // uploadFiles hosts local files on the interactsh server and returns the URLs a // target should fetch. It returns nil when no files were requested. -// -// sessionFile mirrors -session-file: when set, the session is meant to outlive -// this process, so the failure paths below must not deregister it. -func uploadFiles(paths []string, c *client.Client, sessionFile string) []string { - if len(paths) == 0 { +func uploadFiles(c *client.Client, cliOptions *options.CLIClientOptions) []string { + if len(cliOptions.Files) == 0 { return nil } - uploaded, err := c.UploadFiles(paths) + uploaded, err := c.UploadFiles(cliOptions.Files) if err != nil { // Registration already happened -- it has to, since upload support is // only advertised in the register response -- so a session exists on the @@ -357,17 +373,21 @@ func uploadFiles(paths []string, c *client.Client, sessionFile string) []string // for a resumable session, otherwise deregister it. Doing neither would // overstate the server's live session count for every client that trips // this path, and would strand a session the user cannot resume. - if sessionFile != "" { - _ = c.SaveSessionTo(sessionFile) + if cliOptions.SessionFile != "" { + _ = c.SaveSessionTo(cliOptions.SessionFile) } else { _ = c.Close() } + // Name the server in both failures. The client registers with one server + // out of -s, so without it the reader cannot tell which of their servers + // the complaint is about. if errors.Is(err, client.ErrUploadUnsupported) { - gologger.Fatal().Msgf("Server does not accept file uploads; it must be started with -upload\n") + gologger.Fatal().Msgf("Server %s does not accept file uploads; it must be started with -upload%s\n", + c.ServerURL(), electionHint(cliOptions.ServerURL)) } // Fatal rather than a warning: the user asked to host a payload, and // carrying on without it produces a confusing "no interaction" result. - gologger.Fatal().Msgf("Could not upload files: %s\n", err) + gologger.Fatal().Msgf("Could not upload files to %s: %s\n", c.ServerURL(), err) } // One payload host for every file, so the target performs a single DNS diff --git a/cmd/interactsh-client/main_test.go b/cmd/interactsh-client/main_test.go new file mode 100644 index 00000000..92470ee9 --- /dev/null +++ b/cmd/interactsh-client/main_test.go @@ -0,0 +1,36 @@ +package main + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestElectionHint(t *testing.T) { + t.Run("stays silent for a single server", func(t *testing.T) { + // Nothing to disambiguate: the message already names the only server. + require.Empty(t, electionHint("https://oast.pro")) + require.Empty(t, electionHint("")) + // A trailing comma still describes one server. + require.Empty(t, electionHint("https://oast.pro,")) + }) + + t.Run("names the count when several were listed", func(t *testing.T) { + hint := electionHint("oast.pro,oast.live,oast.site") + require.Contains(t, hint, "3 servers") + require.Contains(t, hint, "pass a single server with -file", + "the hint must say what to do, not just what happened") + }) + + t.Run("ignores blank entries and whitespace", func(t *testing.T) { + require.Empty(t, electionHint(" , ")) + require.Contains(t, electionHint("oast.pro, oast.live"), "2 servers") + }) + + t.Run("reads as a suffix to the failure sentence", func(t *testing.T) { + hint := electionHint("a.example,b.example") + require.True(t, strings.HasPrefix(hint, " ("), "must append cleanly after the message") + require.True(t, strings.HasSuffix(hint, ")")) + }) +} From efedfbbdf82916589ed6f434d3c650abeb0cb5e7 Mon Sep 17 00:00:00 2001 From: nisay759 Date: Wed, 5 Aug 2026 18:06:53 +0200 Subject: [PATCH 13/20] docs: regenerate the -h blocks in the README from the binaries Both usage blocks were transcribed by hand and had drifted from the flag sets. Replaced with the verbatim output of interactsh-client -h and interactsh-server -h, with $HOME substituted back into the config paths. This adds the upload flags introduced here -- -fl/-file on the client and the whole UPLOAD group on the server -- and picks up flags that were already shipping but undocumented: -auth, -kai/-keep-alive-interval and -asn on the client, -i as the short form of -ip, and -ru/-rp for the redis backend on the server. Several descriptions and defaults were also stale. The alignment of the client's -asn line is not a typo: its description carries a leading space in the flag definition, so this is what users see. Co-Authored-By: Claude --- README.md | 54 ++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 34 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index b5c452a7..866fd6f5 100644 --- a/README.md +++ b/README.md @@ -59,33 +59,37 @@ Usage: Flags: INPUT: - -s, -server string interactsh server(s) to use (default "oast.pro,oast.live,oast.site,oast.online,oast.fun,oast.me") + -s, -server string interactsh server(s) to use (default "oast.pro,oast.live,oast.site,oast.online,oast.fun,oast.me") + -fl, -file string[] local file(s) to upload and host on the interactsh server CONFIG: -config string flag configuration file (default "$HOME/.config/interactsh-client/config.yaml") + -auth configure projectdiscovery cloud (pdcp) api key (default true) -n, -number int number of interactsh payload to generate (default 1) -t, -token string authentication token to connect protected interactsh server -pi, -poll-interval int poll interval in seconds to pull interaction data (default 5) -nf, -no-http-fallback disable http fallback registration - -cidl, -correlation-id-length int length of the correlation id preamble (min 3, default 20) - -cidn, -correlation-id-nonce-length int length of the correlation id nonce (min 3, default 13) + -cidl, -correlation-id-length int length of the correlation id preamble (min 3, default 20) (default 20) + -cidn, -correlation-id-nonce-length int length of the correlation id nonce (min 3, default 13) (default 13) -sf, -session-file string store/read from session file + -kai, -keep-alive-interval value keep alive interval (default 1m0s) FILTER: -m, -match string[] match interaction based on the specified pattern -f, -filter string[] filter interaction based on the specified pattern -dns-only display only dns interaction in CLI output - -http-only display only http/https interactions in CLI output + -http-only display only http interaction in CLI output -smtp-only display only smtp interactions in CLI output + -asn include asn information of remote ip in json output UPDATE: -up, -update update interactsh-client to latest version -duc, -disable-update-check disable automatic interactsh-client update check - + OUTPUT: -o string output file to write interaction data -json write output in JSON Lines format - -ps, -payload-store enable storing generated interactsh payload to file + -ps, -payload-store write generated interactsh payload to file -psf, -payload-store-file string store generated interactsh payloads to given file (default "interactsh_payload.txt") -v display verbose interaction @@ -353,7 +357,7 @@ Usage: Flags: INPUT: -d, -domain string[] single/multiple configured domain to use for server - -ip string[] public ip address(es) to use for interactsh server (comma-separated,supports both IPv4 & IPv6) + -i, -ip string[] public IP address(es) to use for interactsh server (comma-separated, supports both IPv4 & IPv6) -lip, -listen-ip string public ip address to listen on (default "0.0.0.0") -e, -eviction int number of days to persist interaction data in memory (default 30) -ne, -no-eviction disable periodic data eviction from memory @@ -363,29 +367,31 @@ INPUT: -acao-url string origin url to send in acao header to use web-client) (default "*") -sa, -skip-acme skip acme registration (certificate checks/handshake + TLS protocols will be disabled) -se, -scan-everywhere scan canary token everywhere - -cidl, -correlation-id-length int length of the correlation id preamble (min 3, default 20) - -cidn, -correlation-id-nonce-length int length of the correlation id nonce (min 3, default 13) + -cidl, -correlation-id-length int length of the correlation id preamble (min 3, default 20) (default 20) + -cidn, -correlation-id-nonce-length int length of the correlation id nonce (min 3, default 13) (default 13) -cert string custom certificate path -privkey string custom private key path -oih, -origin-ip-header string HTTP header containing origin ip (interactsh behind a reverse proxy) CONFIG: - -r, -resolvers string[] list of resolvers to use (file or comma separated) - -config string flag configuration file (default "$HOME/.config/interactsh-server/config.yaml") - -dr, -dynamic-resp enable setting up arbitrary response data - -cr, -custom-records string custom dns records YAML file for DNS server - -hi, -http-index string custom index file for http server + -r, -resolvers string[] list of resolvers to use (file or comma separated) + -config string flag configuration file (default "$HOME/.config/interactsh-server/config.yaml") + -dr, -dynamic-resp enable setting up arbitrary response data + -cr, -custom-records string custom dns records YAML file for DNS server + -hi, -http-index string custom index file for http server + -hd, -http-directory string directory with files to serve with http server -dhr, -default-http-response string file to serve for all http requests (takes priority over other options) - -hd, -http-directory string directory with files to serve with http server - -ds, -disk disk based storage - -dsp, -disk-path string disk storage path - -csh, -server-header string custom value of Server header in response - -dv, -disable-version disable publishing interactsh version in response header + -ds, -disk disk based storage + -dsp, -disk-path string disk storage path + -ru, -redis-url string redis connection URL (enables shared state for multi-instance deployments) + -rp, -redis-prefix string redis key prefix (default "interactsh:") + -csh, -server-header string custom value of Server header in response + -dv, -disable-version disable publishing interactsh version in response header UPDATE: -up, -update update interactsh-server to latest version -duc, -disable-update-check disable automatic interactsh-server update check - + SERVICES: -dns-port int port to use for dns service (default 53) -http-port int port to use for http service (default 80) @@ -404,6 +410,14 @@ SERVICES: -ftps-port int port to use for ftps service (default 990) -ftp-dir string ftp directory - temporary if not specified +UPLOAD: + -upload enable client file upload and hosting - self-hosted servers only (authenticated) + -ud, -upload-directory string directory to store uploaded files - temporary if not specified + -umfs, -upload-max-file-size value maximum size of a single uploaded file (default 1mb) + -umf, -upload-max-files int maximum number of uploaded files per session (default 5) + -umts, -upload-max-total-size value maximum total size of all uploaded files on the server (default 1gb) + -ut, -upload-ttl value maximum lifetime of uploaded files (default 24h0m0s) + DEBUG: -version show version of the project -debug start interactsh server in debug mode From 52eb6eaa051f9bc2b1447c6f69920a0fc0fae5e8 Mon Sep 17 00:00:00 2001 From: nisay759 Date: Thu, 6 Aug 2026 14:12:14 +0200 Subject: [PATCH 14/20] docs: state that interactsh prunes a directory in the upload root -upload-directory read as "directory to store uploaded files", which does not warn the operator that interactsh treats part of that directory as its own and deletes from it -- on session end, on -upload-ttl expiry, and at startup, since upload metadata lives only in memory and anything left behind is an orphan. That matters most when the flag points at a directory the operator already uses, or is shared with -ftp-dir, which the feature actively encourages. The flag help now names .interactsh-user-uploads, and the README documents the layout, what is pruned and what is never touched, plus the FTP behaviour that follows from sharing a root: the uploads directory is hidden from listings and refuses to list itself, so the correlation ids with hosted files cannot be enumerated anonymously, while RETR of a known path still works. The -h block in the README is regenerated to match the binary. Co-Authored-By: Claude Opus 5 --- README.md | 18 ++++++++++++++++-- cmd/interactsh-server/main.go | 2 +- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 866fd6f5..aabd7a35 100644 --- a/README.md +++ b/README.md @@ -412,7 +412,7 @@ SERVICES: UPLOAD: -upload enable client file upload and hosting - self-hosted servers only (authenticated) - -ud, -upload-directory string directory to store uploaded files - temporary if not specified + -ud, -upload-directory string directory to host uploaded files from - temporary if not specified; interactsh creates and prunes .interactsh-user-uploads inside it -umfs, -upload-max-file-size value maximum size of a single uploaded file (default 1mb) -umf, -upload-max-files int maximum number of uploaded files per session (default 5) -umts, -upload-max-total-size value maximum total size of all uploaded files on the server (default 1gb) @@ -760,7 +760,7 @@ Server-side options: | Flag | Default | Description | | --- | --- | --- | | `-upload` | off | enable client file upload and hosting | -| `-ud, -upload-directory` | temporary dir | where uploaded files are stored | +| `-ud, -upload-directory` | temporary dir | directory to host uploaded files from; interactsh owns `.interactsh-user-uploads` inside it | | `-umfs, -upload-max-file-size` | `1mb` | maximum size of a single file | | `-umf, -upload-max-files` | `5` | maximum files per session | | `-umts, -upload-max-total-size` | `1gb` | maximum total bytes across all sessions | @@ -794,6 +794,20 @@ Things worth knowing before enabling it: file and the session secret key. Use an `https://` server URL. - The default upload directory is a temporary directory, which on many Linux distributions is memory-backed. Set `-upload-directory` explicitly on a real deployment. +- **Interactsh creates and prunes one directory inside the upload root.** Hosted files are laid out as + `/.interactsh-user-uploads//`, and everything under + `.interactsh-user-uploads` is deleted when its session ends, when `-upload-ttl` expires it, and at + startup — upload metadata lives only in memory, so nothing there survives a restart. The rest of the + root is never touched, which is what makes it safe to point `-upload-directory` at a directory you + already use, or to share it with `-ftp-dir`. +- With `-ftp` and no `-ftp-dir`, the FTP root is set to the upload root so that hosted files are + reachable over FTP with no extra configuration. If you set both flags they must name the same + directory, otherwise FTP cannot see the uploads: the server reports the mismatch at startup and stops + offering `ftp://` URLs to clients, so hosting degrades to HTTP only rather than handing out FTP URLs + that resolve to nothing. The uploads directory is hidden from FTP listings — `LIST /` shows your own content but + not `.interactsh-user-uploads`, and that directory refuses to list its own contents, so an anonymous + client cannot enumerate the correlation IDs that currently have hosted files. `RETR` of a known path + works, which is what the payload URL relies on. ## Dynamic HTTP Response diff --git a/cmd/interactsh-server/main.go b/cmd/interactsh-server/main.go index 31dce010..eb644d7b 100644 --- a/cmd/interactsh-server/main.go +++ b/cmd/interactsh-server/main.go @@ -107,7 +107,7 @@ func main() { flagSet.CreateGroup("upload", "Upload", flagSet.BoolVar(&cliOptions.Upload, "upload", false, "enable client file upload and hosting - self-hosted servers only (authenticated)"), - flagSet.StringVarP(&cliOptions.UploadDirectory, "upload-directory", "ud", "", "directory to store uploaded files - temporary if not specified"), + flagSet.StringVarP(&cliOptions.UploadDirectory, "upload-directory", "ud", "", "directory to host uploaded files from - temporary if not specified; interactsh creates and prunes .interactsh-user-uploads inside it"), flagSet.SizeVarP(&cliOptions.UploadMaxFileSize, "upload-max-file-size", "umfs", "1mb", "maximum size of a single uploaded file"), flagSet.IntVarP(&cliOptions.UploadMaxFiles, "upload-max-files", "umf", 5, "maximum number of uploaded files per session"), flagSet.SizeVarP(&cliOptions.UploadMaxTotalSize, "upload-max-total-size", "umts", "1gb", "maximum total size of all uploaded files on the server"), From d8cd700d08b93bd3434856318aa9a37b888adbcc Mon Sep 17 00:00:00 2001 From: nisay759 Date: Thu, 6 Aug 2026 14:52:20 +0200 Subject: [PATCH 15/20] server: stage uploads and commit the batch together UpdateUploads runs the batch under the correlation id's lock and discards the metadata update when the callback fails, so the metadata side was transactional. The disk side was not: Save reserved quota, wrote a temp file and renamed it into place before returning, so by the time a later file in the same request failed the earlier ones were already committed and already charged. The result was a file on disk holding server-wide quota that nothing could reach -- serveUploadedFile consults the metadata first, so it answered 404 -- and that nothing could reclaim, because the retry read existingSize from the rolled-back metadata, saw 0, and asked for the space a second time. Reproduced with a 1024-byte cap and two 1000-byte files: 507, no metadata, a.bin on disk, 1000 bytes charged, and every later upload on every session refused until the session ended or -upload-ttl expired. The orphan stayed readable over FTP, which serves the filesystem with no metadata check. Two ordinary failures reach that path: the per-session file cap, checked cumulatively inside the callback while pre-validation only checks the request, and the global quota. Filesystem errors do too. Save splits along the seam it already had. Stage validates, reserves and writes the bytes under a temporary name, reachable by nobody. Commit renames a staged file into place, and the batch calls it only once every file has staged. Abort removes a staged file and releases its reservation, deferred so it runs on a panic as well. Save itself becomes Stage plus Commit, for single-file callers with nothing to unwind. Unwinding by deleting what had already been written would not have been enough: for a name that already existed the previous content is gone the moment the rename lands, so a compensating delete turns a leaked file into lost data. Staging avoids the question, since the original stays untouched until the whole batch is ready. One residual is accepted. If a Commit fails after earlier ones in the batch have succeeded, those are published while the metadata is discarded -- but that is a rename failing on a file just written into the same directory, so it means something severe. Abort unwinds what it safely can: a committed file that overwrote nothing is removed, one that replaced an existing file is left with a warning, so the exposure is one rename rather than a whole batch. Everything Abort touches resolves to //, so unwinding one session can never reach another's files. Co-Authored-By: Claude Opus 5 --- pkg/server/upload.go | 132 ++++++++++++++++++---- pkg/server/upload_batch_test.go | 191 ++++++++++++++++++++++++++++++++ pkg/server/upload_handler.go | 36 +++++- 3 files changed, 334 insertions(+), 25 deletions(-) create mode 100644 pkg/server/upload_batch_test.go diff --git a/pkg/server/upload.go b/pkg/server/upload.go index 4d0e8c8e..b7b04977 100644 --- a/pkg/server/upload.go +++ b/pkg/server/upload.go @@ -253,64 +253,154 @@ func (s *UploadStore) sessionDir(correlationID string) string { return filepath.Join(s.sessionsRoot, correlationID) } -// Save writes one file for a correlation ID and returns its size and digest. +// stagedUpload is a file written into its session directory under a temporary +// name, with its quota already reserved, waiting to be renamed into place. +// +// Staging exists so that a multi-file upload is all-or-nothing. Writing straight +// to the final name would commit each file as it went, and a failure part-way +// through a batch -- the per-session file cap, the global quota, a full disk -- +// left the earlier files on disk holding quota while the caller discarded the +// metadata that made them reachable. +type stagedUpload struct { + correlationID string + // name is the final name; tmp is where the bytes are until Commit. + name string + tmp string + size int64 + // delta is the quota reserved for this file: the net change, since + // overwriting a name replaces the bytes it already occupied. + delta int64 + sha256 string + // overwrites records that the final name already held a file, which is what + // makes discarding a committed file unsafe: the previous content is gone. + overwrites bool + committed bool +} + +// Stage validates one file, reserves its quota and writes it under a temporary +// name in its session directory. Nothing is reachable until Commit. // // It is called from inside Storage.UpdateUploads, i.e. under the correlation // ID's lock, so the caller's quota check and this write are atomic with respect // to other uploads for the same session. -func (s *UploadStore) Save(correlationID, name string, data []byte, existingSize int64) (int64, string, error) { +func (s *UploadStore) Stage(correlationID, name string, data []byte, existingSize int64) (*stagedUpload, error) { if !isSafeUploadName(name) { - return 0, "", uploadErr(UploadErrBadName, "invalid file name %q", name) + return nil, uploadErr(UploadErrBadName, "invalid file name %q", name) } size := int64(len(data)) if size == 0 { - return 0, "", uploadErr(UploadErrBadName, "file %q is empty", name) + return nil, uploadErr(UploadErrBadName, "file %q is empty", name) } if size > s.maxFileSize { - return 0, "", uploadErr(UploadErrTooLarge, "file %q is %d bytes, limit is %d", name, size, s.maxFileSize) + return nil, uploadErr(UploadErrTooLarge, "file %q is %d bytes, limit is %d", name, size, s.maxFileSize) } dir := s.sessionDir(correlationID) if dir == "" { - return 0, "", uploadErr(UploadErrOther, "invalid correlation-id") + return nil, uploadErr(UploadErrOther, "invalid correlation-id") } // Reserve space before writing. existingSize is what this name already // occupies, since overwriting replaces rather than adds. - if delta := size - existingSize; delta > 0 { + delta := size - existingSize + if delta > 0 { if s.totalBytes.Add(delta) > s.maxTotal { s.totalBytes.Add(-delta) - return 0, "", uploadErr(UploadErrOutOfSpace, "server upload capacity exhausted") + return nil, uploadErr(UploadErrOutOfSpace, "server upload capacity exhausted") } } else { s.totalBytes.Add(delta) } + release := func() { s.totalBytes.Add(-delta) } + if err := os.MkdirAll(dir, uploadSessionDirPerm); err != nil { - s.totalBytes.Add(existingSize - size) - return 0, "", errors.Wrap(err, "could not create session directory") + release() + return nil, errors.Wrap(err, "could not create session directory") } // Plain os rather than the rooted handle: os.Root has no Rename before Go // 1.25, and both components are already constrained -- the correlation ID // is alphanumeric and length-checked, the name passed the allowlist, so // neither can contain a separator or traversal sequence. - // - // Write to a temp name and rename into place, so a reader (HTTP or the FTP - // file driver) never observes a partially written file. tmp := filepath.Join(dir, ".upload-"+xid.New().String()) if err := os.WriteFile(tmp, data, uploadFilePerm); err != nil { - s.totalBytes.Add(existingSize - size) - return 0, "", errors.Wrap(err, "could not write uploaded file") - } - if err := os.Rename(tmp, filepath.Join(dir, name)); err != nil { - _ = os.Remove(tmp) - s.totalBytes.Add(existingSize - size) - return 0, "", errors.Wrap(err, "could not commit uploaded file") + release() + return nil, errors.Wrap(err, "could not write uploaded file") } sum := sha256.Sum256(data) - return size, hex.EncodeToString(sum[:]), nil + return &stagedUpload{ + correlationID: correlationID, + name: name, + tmp: tmp, + size: size, + delta: delta, + sha256: hex.EncodeToString(sum[:]), + overwrites: existingSize > 0, + }, nil +} + +// Commit renames a staged file into place, making it reachable. Writing to a +// temporary name and renaming is also what stops a reader -- the HTTP handler or +// the FTP file driver -- from ever observing a partially written file. +func (s *UploadStore) Commit(st *stagedUpload) error { + dir := s.sessionDir(st.correlationID) + if dir == "" { + return uploadErr(UploadErrOther, "invalid correlation-id") + } + if err := os.Rename(st.tmp, filepath.Join(dir, st.name)); err != nil { + return errors.Wrap(err, "could not commit uploaded file") + } + st.committed = true + return nil +} + +// Abort discards a staged file and releases its reservation. +// +// A file that was never committed is only a temp file, so it goes without +// question. A committed one is a narrower case -- a later rename in the same +// batch failed -- and is only removed when it did not overwrite anything: the +// previous content of an overwritten name is already gone, so deleting it would +// turn a leaked file into lost data. Everything Abort touches resolves to +// //, so it can never reach another +// session's files. +func (s *UploadStore) Abort(st *stagedUpload) { + if !st.committed { + if err := os.Remove(st.tmp); err != nil && !os.IsNotExist(err) { + gologger.Debug().Msgf("Could not remove staged upload %s: %s\n", st.tmp, err) + } + s.totalBytes.Add(-st.delta) + return + } + if st.overwrites { + gologger.Warning().Msgf("Uploaded file %s/%s replaced an existing file and cannot be rolled back\n", + st.correlationID, st.name) + return + } + dir := s.sessionDir(st.correlationID) + if dir == "" { + return + } + if err := os.Remove(filepath.Join(dir, st.name)); err != nil && !os.IsNotExist(err) { + gologger.Debug().Msgf("Could not roll back uploaded file %s/%s: %s\n", st.correlationID, st.name, err) + return + } + s.totalBytes.Add(-st.delta) +} + +// Save stages and immediately commits one file, for callers handling a single +// file with nothing to unwind. +func (s *UploadStore) Save(correlationID, name string, data []byte, existingSize int64) (int64, string, error) { + st, err := s.Stage(correlationID, name, data, existingSize) + if err != nil { + return 0, "", err + } + if err := s.Commit(st); err != nil { + s.Abort(st) + return 0, "", err + } + return st.size, st.sha256, nil } // Open returns a readable handle to an uploaded file. The name is re-validated diff --git a/pkg/server/upload_batch_test.go b/pkg/server/upload_batch_test.go new file mode 100644 index 00000000..dd817843 --- /dev/null +++ b/pkg/server/upload_batch_test.go @@ -0,0 +1,191 @@ +package server + +import ( + "encoding/base64" + "encoding/json" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/google/uuid" + "github.com/rs/xid" + "github.com/stretchr/testify/require" +) + +// orderedUploadBody is uploadBody with a defined file order, which the map-based +// helper cannot give: these tests need to know which file fails. +func orderedUploadBody(t *testing.T, correlationID, secret string, names []string, data [][]byte) string { + t.Helper() + require.Len(t, data, len(names)) + + req := UploadRequest{CorrelationID: correlationID, SecretKey: secret} + for i, name := range names { + req.Files = append(req.Files, UploadFileRequest{ + Name: name, + Data: base64.StdEncoding.EncodeToString(data[i]), + }) + } + encoded, err := json.Marshal(req) + require.NoError(t, err) + return string(encoded) +} + +// sessionFiles lists the file names on disk for a session, temp files included, +// so that a leaked staging file is visible to these assertions. +func sessionFiles(t *testing.T, store *UploadStore, correlationID string) []string { + t.Helper() + entries, err := os.ReadDir(store.sessionDir(correlationID)) + if os.IsNotExist(err) { + return nil + } + require.NoError(t, err) + out := make([]string, 0, len(entries)) + for _, e := range entries { + out = append(out, e.Name()) + } + return out +} + +// newSession registers an additional session on an existing server, for the +// cross-session assertions. +func newSession(t *testing.T, h *HTTPServer) (correlationID, secret string) { + t.Helper() + correlationID = xid.New().String() + secret = uuid.New().String() + require.NoError(t, h.options.Storage.SetIDPublicKey(correlationID, secret, testPublicKey(t))) + return correlationID, secret +} + +// A multi-file upload is all-or-nothing. Committing each file as it is written +// left the earlier ones on disk holding quota while the metadata that made them +// reachable was discarded: unusable, uncollectable until the session ended, and +// counted against every other session's uploads. +func TestUploadBatchIsAtomic(t *testing.T) { + t.Run("quota exhausted part-way leaves nothing behind", func(t *testing.T) { + h, id, secret := uploadTestServer(t, true) + store := h.options.UploadStore + store.maxTotal = 1500 // fits one 1000-byte file, not two + + body := orderedUploadBody(t, id, secret, + []string{"a.bin", "b.bin"}, + [][]byte{make([]byte, 1000), make([]byte, 1000)}) + resp := doUpload(t, h, body) + require.Equal(t, http.StatusInsufficientStorage, resp.StatusCode) + + files, _ := h.options.UploadStorage().ListUploads(id) + require.Empty(t, files, "metadata must be unchanged") + require.Empty(t, sessionFiles(t, store, id), "no file, and no staging temp file, may survive") + require.Zero(t, store.totalBytes.Load(), "the reservation must be released") + }) + + t.Run("and the same upload then succeeds on retry", func(t *testing.T) { + h, id, secret := uploadTestServer(t, true) + store := h.options.UploadStore + store.maxTotal = 1500 + + require.Equal(t, http.StatusInsufficientStorage, doUpload(t, h, orderedUploadBody(t, id, secret, + []string{"a.bin", "b.bin"}, + [][]byte{make([]byte, 1000), make([]byte, 1000)})).StatusCode) + + // Previously wedged: the orphan still held 1000 of the 1500 bytes while + // metadata reported nothing, so this retry was refused indefinitely. + resp := doUpload(t, h, orderedUploadBody(t, id, secret, + []string{"a.bin"}, [][]byte{make([]byte, 1000)})) + require.Equal(t, http.StatusOK, resp.StatusCode, "a retry of the file that fitted must succeed") + + files, _ := h.options.UploadStorage().ListUploads(id) + require.Len(t, files, 1) + require.EqualValues(t, 1000, store.totalBytes.Load(), "charged once, not twice") + }) + + t.Run("per-session file cap reached part-way leaves nothing behind", func(t *testing.T) { + h, id, secret := uploadTestServer(t, true) // UploadMaxFiles is 3 + store := h.options.UploadStore + + for _, name := range []string{"f0.dtd", "f1.dtd"} { + require.Equal(t, http.StatusOK, + doUpload(t, h, orderedUploadBody(t, id, secret, []string{name}, [][]byte{[]byte("x")})).StatusCode) + } + before := store.totalBytes.Load() + + // Two more against a limit of three: the first fits, the second does not. + resp := doUpload(t, h, orderedUploadBody(t, id, secret, + []string{"f2.dtd", "f3.dtd"}, [][]byte{[]byte("yy"), []byte("zz")})) + require.Equal(t, http.StatusRequestEntityTooLarge, resp.StatusCode) + + files, _ := h.options.UploadStorage().ListUploads(id) + require.Len(t, files, 2, "metadata must be unchanged") + require.ElementsMatch(t, []string{"f0.dtd", "f1.dtd"}, sessionFiles(t, store, id), + "the file that fitted must not be left on disk") + require.Equal(t, before, store.totalBytes.Load()) + }) + + // The reason staging beats deleting what was already written: for a name that + // already existed, the previous content is gone the moment it is overwritten, + // so a compensating delete would turn a leaked file into lost data. + t.Run("a failed batch does not disturb the file it would have replaced", func(t *testing.T) { + h, id, secret := uploadTestServer(t, true) + store := h.options.UploadStore + + original := []byte("original payload") + require.Equal(t, http.StatusOK, + doUpload(t, h, orderedUploadBody(t, id, secret, []string{"a.dtd"}, [][]byte{original})).StatusCode) + before, _ := h.options.UploadStorage().ListUploads(id) + require.Len(t, before, 1) + charged := store.totalBytes.Load() + + // Replace a.dtd and add a file that cannot fit: the batch must fail whole. + store.maxTotal = int64(len(original)) + 10 + resp := doUpload(t, h, orderedUploadBody(t, id, secret, + []string{"a.dtd", "b.dtd"}, + [][]byte{[]byte("replacement payload"), make([]byte, 1000)})) + require.NotEqual(t, http.StatusOK, resp.StatusCode) + + onDisk, err := os.ReadFile(filepath.Join(store.sessionDir(id), "a.dtd")) + require.NoError(t, err) + require.Equal(t, original, onDisk, "the previously hosted file must be untouched") + + after, _ := h.options.UploadStorage().ListUploads(id) + require.Equal(t, before, after, "metadata must still describe the file that is on disk") + require.Equal(t, charged, store.totalBytes.Load()) + require.ElementsMatch(t, []string{"a.dtd"}, sessionFiles(t, store, id)) + }) + + // Unwinding resolves every path through sessionDir plus a validated name, so + // it cannot reach outside the session it belongs to. + t.Run("unwinding one session does not touch another", func(t *testing.T) { + h, mine, mySecret := uploadTestServer(t, true) + store := h.options.UploadStore + + theirs, theirSecret := newSession(t, h) + require.Equal(t, http.StatusOK, doUpload(t, h, + orderedUploadBody(t, theirs, theirSecret, []string{"a.bin"}, [][]byte{[]byte("their payload")})).StatusCode) + + store.maxTotal = store.totalBytes.Load() + 1200 + resp := doUpload(t, h, orderedUploadBody(t, mine, mySecret, + []string{"a.bin", "b.bin"}, + [][]byte{make([]byte, 1000), make([]byte, 1000)})) + require.Equal(t, http.StatusInsufficientStorage, resp.StatusCode) + + theirFile, err := os.ReadFile(filepath.Join(store.sessionDir(theirs), "a.bin")) + require.NoError(t, err) + require.Equal(t, []byte("their payload"), theirFile, + "another session's identically named file must be untouched") + theirMeta, ok := h.options.UploadStorage().ListUploads(theirs) + require.True(t, ok) + require.Len(t, theirMeta, 1) + require.Empty(t, sessionFiles(t, store, mine)) + }) + + t.Run("no staging temp files survive a successful batch either", func(t *testing.T) { + h, id, secret := uploadTestServer(t, true) + require.Equal(t, http.StatusOK, doUpload(t, h, orderedUploadBody(t, id, secret, + []string{"a.dtd", "b.dtd"}, [][]byte{[]byte("one"), []byte("two")})).StatusCode) + + for _, name := range sessionFiles(t, h.options.UploadStore, id) { + require.False(t, strings.HasPrefix(name, ".upload-"), "temp file %q left behind", name) + } + }) +} diff --git a/pkg/server/upload_handler.go b/pkg/server/upload_handler.go index 4320a1e8..c4d2d4ae 100644 --- a/pkg/server/upload_handler.go +++ b/pkg/server/upload_handler.go @@ -185,6 +185,21 @@ func (h *HTTPServer) uploadHandler(w http.ResponseWriter, req *http.Request) { ftpPrefix = path.Join("/", uploadsDirName, r.CorrelationID) ) + // Files are staged first and committed only once every one of them has been + // written, so a failure part-way through a batch leaves nothing reachable and + // nothing charged. Committing as we went would leave the earlier files on + // disk holding quota while UpdateUploads discarded the metadata that made + // them reachable -- unusable, uncollectable until the session ended, and + // still counted against every other session's uploads. + // + // Anything still staged when this returns is unwound, including on a panic. + var staged []*stagedUpload + defer func() { + for _, st := range staged { + store.Abort(st) + } + }() + // The disk writes happen inside UpdateUploads, i.e. under the correlation // ID's lock, so the per-session quota check and the commit are atomic with // respect to a concurrent upload for the same session. @@ -206,12 +221,13 @@ func (h *HTTPServer) uploadHandler(w http.ResponseWriter, req *http.Request) { "session already holds %d files, limit is %d", len(updated), store.MaxFiles()) } - size, sum, err := store.Save(r.CorrelationID, f.Name, decoded[i], existingSize) + st, err := store.Stage(r.CorrelationID, f.Name, decoded[i], existingSize) if err != nil { return nil, err } + staged = append(staged, st) - record := storage.UploadedFile{Name: f.Name, Size: size, SHA256: sum, Timestamp: time.Now()} + record := storage.UploadedFile{Name: f.Name, Size: st.size, SHA256: st.sha256, Timestamp: time.Now()} if idx == -1 { updated = append(updated, record) } else { @@ -219,12 +235,22 @@ func (h *HTTPServer) uploadHandler(w http.ResponseWriter, req *http.Request) { } response = append(response, UploadedFileResponse{ Name: f.Name, - Size: size, - SHA256: sum, + Size: st.size, + SHA256: st.sha256, HTTPPath: path.Join("/f", f.Name), FTPPath: path.Join(ftpPrefix, f.Name), }) } + + // Every file is on disk under a temporary name; publish them together. + // A rename failing here is severe -- the write into the same directory + // has already succeeded -- so the batch still fails, and Abort unwinds + // what it safely can. + for _, st := range staged { + if err := store.Commit(st); err != nil { + return nil, err + } + } return updated, nil }) @@ -232,6 +258,8 @@ func (h *HTTPServer) uploadHandler(w http.ResponseWriter, req *http.Request) { h.writeUploadError(w, err) return } + // Committed and recorded: there is nothing left to unwind. + staged = nil gologger.Debug().Msgf("Stored %d uploaded file(s) for %s\n", len(response), r.CorrelationID) w.Header().Set("Content-Type", "application/json; charset=utf-8") From 933bc63f5edc623476cc26f8859b83e2ce3f10f3 Mon Sep 17 00:00:00 2001 From: nisay759 Date: Thu, 6 Aug 2026 15:47:18 +0200 Subject: [PATCH 16/20] client: keep payload hostnames and hosted file URLs in separate files -psf is a machine interface: payload hostnames, one per line, exactly -n of them, which is what a wrapper script substituting each line into a payload template relies on. Appending hosted-file URLs to it broke both halves of that contract at once -- the line count stopped matching -n, and "$line" became a full URL, so a template produced http://https://host/f/evil.dtd/ and a resolver lookup simply failed. Nothing reported an error, because the file still parsed as lines. Hosted-file URLs are worth having in a file, so they get their own: -fsf, -file-store-file, empty by default and enabled by being set, as -o is. Each file now holds one record type. Both files are newline-terminated. Previously the last record had no terminator, so wc -l reported one fewer record than the file held and a plain "while read line" loop dropped it -- discarding a payload silently. That predates this branch but the appended URLs changed which record got swallowed, and the files exist to be read by exactly that idiom. Co-Authored-By: Claude Opus 5 --- README.md | 1 + cmd/interactsh-client/main.go | 31 +++++++++++++++-- cmd/interactsh-client/main_test.go | 55 ++++++++++++++++++++++++++++++ pkg/options/client_options.go | 1 + 4 files changed, 86 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index aabd7a35..9fcd7911 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,7 @@ OUTPUT: -json write output in JSON Lines format -ps, -payload-store write generated interactsh payload to file -psf, -payload-store-file string store generated interactsh payloads to given file (default "interactsh_payload.txt") + -fsf, -file-store-file string store hosted file URLs to given file (requires -file) -v display verbose interaction DEBUG: diff --git a/cmd/interactsh-client/main.go b/cmd/interactsh-client/main.go index 0518a9d2..712d4299 100644 --- a/cmd/interactsh-client/main.go +++ b/cmd/interactsh-client/main.go @@ -84,6 +84,7 @@ func main() { flagSet.BoolVar(&cliOptions.JSON, "json", false, "write output in JSON Lines format"), flagSet.BoolVarP(&cliOptions.StorePayload, "payload-store", "ps", false, "write generated interactsh payload to file"), flagSet.StringVarP(&cliOptions.StorePayloadFile, "payload-store-file", "psf", settings.StorePayloadFileDefault, "store generated interactsh payloads to given file"), + flagSet.StringVarP(&cliOptions.FileStoreFile, "file-store-file", "fsf", "", "store hosted file URLs to given file (requires -file)"), flagSet.BoolVar(&cliOptions.Verbose, "v", false, "display verbose interaction"), ) @@ -197,12 +198,24 @@ func main() { } } + // One record type per file. -psf is a machine-readable list of payload + // hostnames, one per line and exactly -n of them, which is what a wrapper + // script substituting into a payload template relies on; mixing hosted-file + // URLs into it turns "$line" into "https://host/f/x" and silently produces + // nonsense like http://https://host/f/x/. Hosted-file URLs get their own file. if cliOptions.StorePayload && cliOptions.StorePayloadFile != "" { - stored := append(append([]string{}, interactshURLs...), fileURLs...) - if err := os.WriteFile(cliOptions.StorePayloadFile, []byte(strings.Join(stored, "\n")), 0644); err != nil { + if err := writeLines(cliOptions.StorePayloadFile, interactshURLs); err != nil { gologger.Fatal().Msgf("Could not write to payload output file: %s\n", err) } } + if cliOptions.FileStoreFile != "" { + if len(fileURLs) == 0 { + gologger.Warning().Msgf("-file-store-file was given without -file, so no hosted file URLs were written\n") + } + if err := writeLines(cliOptions.FileStoreFile, fileURLs); err != nil { + gologger.Fatal().Msgf("Could not write to file URL output file: %s\n", err) + } + } // show all interactions noFilter := !cliOptions.DNSOnly && !cliOptions.HTTPOnly && !cliOptions.SmtpOnly @@ -443,3 +456,17 @@ func (m *regexMatcher) match(item string) bool { } return false } + +// writeLines writes one record per line, newline-terminated. +// +// The terminator matters: without it the last record has no newline, so a plain +// "while read line" loop -- the most likely consumer of these files -- drops it, +// and wc -l reports one fewer record than the file holds. +func writeLines(path string, lines []string) error { + var b strings.Builder + for _, line := range lines { + b.WriteString(line) + b.WriteString("\n") + } + return os.WriteFile(path, []byte(b.String()), 0644) +} diff --git a/cmd/interactsh-client/main_test.go b/cmd/interactsh-client/main_test.go index 92470ee9..1e314451 100644 --- a/cmd/interactsh-client/main_test.go +++ b/cmd/interactsh-client/main_test.go @@ -1,6 +1,8 @@ package main import ( + "os" + "path/filepath" "strings" "testing" @@ -34,3 +36,56 @@ func TestElectionHint(t *testing.T) { require.True(t, strings.HasSuffix(hint, ")")) }) } + +// -psf is a machine interface: one payload hostname per line, exactly -n of them. +// A consumer substitutes each line into a payload template, so a line carrying a +// full URL produces nonsense, and a missing trailing newline costs it the last +// record. +func TestWriteLines(t *testing.T) { + t.Run("one record per line, newline terminated", func(t *testing.T) { + path := filepath.Join(t.TempDir(), "payloads.txt") + payloads := []string{ + "c6rj61aciaeutn2ae680ti6cc3rxeenc3.oast.pro", + "c6rj61aciaeutn2ae680xk4tqy8pqhwmi.oast.pro", + } + require.NoError(t, writeLines(path, payloads)) + + raw, err := os.ReadFile(path) + require.NoError(t, err) + require.Equal(t, strings.Join(payloads, "\n")+"\n", string(raw)) + + // What "while read" and "wc -l" see, which is the point of the terminator. + require.Equal(t, len(payloads), strings.Count(string(raw), "\n")) + require.Equal(t, payloads, strings.Split(strings.TrimSuffix(string(raw), "\n"), "\n")) + }) + + t.Run("no record is a valid empty file", func(t *testing.T) { + path := filepath.Join(t.TempDir(), "empty.txt") + require.NoError(t, writeLines(path, nil)) + raw, err := os.ReadFile(path) + require.NoError(t, err) + require.Empty(t, raw, "an empty list must not leave a stray newline") + }) + + t.Run("hosted file URLs stay in their own file", func(t *testing.T) { + dir := t.TempDir() + payloads := []string{"c6rj61aciaeutn2ae680ti6cc3rxeenc3.oast.pro"} + fileURLs := []string{ + "https://c6rj61aciaeutn2ae680xk4tqy8pqhwmi.oast.pro/f/evil.dtd", + "ftp://c6rj61aciaeutn2ae680xk4tqy8pqhwmi.oast.pro/.interactsh-user-uploads/c6rj61aciaeutn2ae680/evil.dtd", + } + payloadFile := filepath.Join(dir, "payloads.txt") + urlFile := filepath.Join(dir, "files.txt") + require.NoError(t, writeLines(payloadFile, payloads)) + require.NoError(t, writeLines(urlFile, fileURLs)) + + gotPayloads, err := os.ReadFile(payloadFile) + require.NoError(t, err) + require.NotContains(t, string(gotPayloads), "://", + "a payload file line must be a hostname, never a URL") + + gotURLs, err := os.ReadFile(urlFile) + require.NoError(t, err) + require.Equal(t, strings.Join(fileURLs, "\n")+"\n", string(gotURLs)) + }) +} diff --git a/pkg/options/client_options.go b/pkg/options/client_options.go index c6e8c804..f8c7333b 100644 --- a/pkg/options/client_options.go +++ b/pkg/options/client_options.go @@ -18,6 +18,7 @@ type CLIClientOptions struct { JSON bool StorePayload bool StorePayloadFile string + FileStoreFile string Verbose bool PollInterval int DNSOnly bool From 2f3abef53c4fadfefafb24c0e38740f8e1b78774 Mon Sep 17 00:00:00 2001 From: nisay759 Date: Thu, 6 Aug 2026 16:02:15 +0200 Subject: [PATCH 17/20] client: stop reading 404 as "uploads unsupported", name version skew The client mapped 404 and 405 to ErrUploadUnsupported, which the CLI renders as "does not accept file uploads; it must be started with -upload". But the server answers 404 for an unknown correlation id: a session that has been evicted or was never registered, whose remedy is to re-register, on a server that already has -upload and said so in its capabilities a moment earlier. The advice named a flag the operator had already passed. Nor did that mapping serve the case it looks written for. A server predating /upload has no route for it, so the request reaches the catch-all and defaultHandler answers 200 with HTML -- checked against a build of this branch's base commit -- never 404 or 405. So version skew fell through to the JSON decode and failed with `invalid character '<' looking for beginning of value`. Both are now handled where the information actually is. A server that advertised no capabilities at all predates the feature, so UploadFiles refuses before sending anything, with ErrUploadNotAdvertised and a message that says to upgrade the server. 404 and 405 fall through to the default branch, which reports the status and the server's own reason -- "404 Not Found: unknown correlation-id" -- which is true whatever the cause. 501 stays as a backstop for a server that advertised uploads and then refused them, reachable behind a mismatched load balancer. ErrUploadNotAdvertised wraps ErrUploadUnsupported, so a library caller asking only whether hosting is possible is unaffected; pkg/client is consumed that way. The wrap is one-way, which is why the base sentinel is a plain error rather than an errkit one: errkit compares by message, so a wrapped errkit sentinel would satisfy errors.Is in both directions and send a server with -upload merely switched off down the "upgrade the server" path. Absence of capabilities only means "predates the feature" when a registration actually completed. Resuming a session (-sf) re-registers, but the server refuses a duplicate registration while the session is still alive and that error is deliberately ignored, so no capabilities are ever received -- reading that nil as "the server advertised nothing" would make every resume refuse to upload, blaming a server that hosts files perfectly well. capabilitiesKnown records that an answer was received, so an unknown capability set attempts the upload and lets the response speak, while a known-empty one fails closed. The two tests covering the old mapping passed nil capabilities, so after this change they would have short-circuited before sending a request and still passed on the wrapped error, asserting nothing. They now advertise capabilities and check that the request was actually made. Co-Authored-By: Claude Opus 5 --- cmd/interactsh-client/main.go | 4 ++ pkg/client/client.go | 17 ++++- pkg/client/upload.go | 34 +++++++++- pkg/client/upload_test.go | 114 +++++++++++++++++++++++++++++++--- 4 files changed, 155 insertions(+), 14 deletions(-) diff --git a/cmd/interactsh-client/main.go b/cmd/interactsh-client/main.go index 712d4299..e4f07ec2 100644 --- a/cmd/interactsh-client/main.go +++ b/cmd/interactsh-client/main.go @@ -394,6 +394,10 @@ func uploadFiles(c *client.Client, cliOptions *options.CLIClientOptions) []strin // Name the server in both failures. The client registers with one server // out of -s, so without it the reader cannot tell which of their servers // the complaint is about. + if errors.Is(err, client.ErrUploadNotAdvertised) { + gologger.Fatal().Msgf("Server %s did not advertise file hosting, so it predates -file; upgrade the server%s\n", + c.ServerURL(), electionHint(cliOptions.ServerURL)) + } if errors.Is(err, client.ErrUploadUnsupported) { gologger.Fatal().Msgf("Server %s does not accept file uploads; it must be started with -upload%s\n", c.ServerURL(), electionHint(cliOptions.ServerURL)) diff --git a/pkg/client/client.go b/pkg/client/client.go index e5f35a53..07084abe 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -67,6 +67,12 @@ type Client struct { token string correlationIdLength int CorrelationIdNonceLength int + // capabilitiesKnown records that a registration response was received and + // parsed, which is what makes the absence of a capabilities block meaningful. + // A resumed session whose re-registration was refused because the session is + // still alive never learns what the server offers, and "unknown" must not be + // read as "the server offers nothing". + capabilitiesKnown atomic.Bool // capabilities holds the *server.Capabilities advertised at registration. // Written from performRegistration, which the keep-alive goroutine also // calls, hence atomic.Value rather than a bare field. @@ -647,8 +653,10 @@ func (c *Client) performRegistration(serverURL string, payload []byte) error { return fmt.Errorf("could not get register response: %s", response.Message) } - // Nil against a server predating capability advertisement, which callers - // treat as "unknown" rather than "unsupported". + // A successful registration is authoritative about what the server offers, + // including the absence of a capabilities block, which means the server + // predates them. + c.capabilitiesKnown.Store(true) if response.Capabilities != nil { c.capabilities.Store(response.Capabilities) } @@ -658,6 +666,11 @@ func (c *Client) performRegistration(serverURL string, payload []byte) error { return nil } +// CapabilitiesKnown reports whether a registration response has been received, +// which is what distinguishes a server that advertised no capabilities from a +// resumed session that never got to ask. +func (c *Client) CapabilitiesKnown() bool { return c.capabilitiesKnown.Load() } + // Capabilities returns the optional features advertised by the server at // registration, or nil if the server did not advertise any. func (c *Client) Capabilities() *server.Capabilities { diff --git a/pkg/client/upload.go b/pkg/client/upload.go index 83c39345..39687543 100644 --- a/pkg/client/upload.go +++ b/pkg/client/upload.go @@ -29,6 +29,15 @@ import ( // message contains it match in both directions. var ErrUploadUnsupported = errors.New("interactsh server does not support file upload") +// ErrUploadNotAdvertised is returned when the server advertised no capabilities +// at all, which means it predates file hosting. The remedy is to upgrade the +// server rather than to pass it a flag, so it is distinguishable from a server +// that advertised uploads as switched off. +// +// It wraps ErrUploadUnsupported, so a caller that only asks "can this server +// host files?" needs no change. +var ErrUploadNotAdvertised = fmt.Errorf("%w: server did not advertise file hosting capabilities", ErrUploadUnsupported) + // defaultMaxUploadFileSize bounds a local file when the server has not told us // its limit, so a mistyped -file cannot try to push a huge file over the wire. const defaultMaxUploadFileSize = 1 << 20 @@ -66,9 +75,18 @@ func (c *Client) UploadFiles(paths []string) ([]UploadedFile, error) { return nil, errkit.New("client is not registered with any server") } - // Fail closed when the server has told us it cannot host files. - if caps := c.Capabilities(); caps != nil && !caps.Upload { + // Fail closed rather than send a request that cannot succeed, but only where + // the answer is known. A registration that produced no capabilities block is + // authoritative -- that server predates the feature, and its catch-all + // answers 200 with HTML, which would otherwise surface as an opaque JSON + // decode error. A resumed session whose re-registration was refused because + // the session is still alive knows nothing either way, so it must attempt the + // upload and let the response speak. + switch caps := c.Capabilities(); { + case caps != nil && !caps.Upload: return nil, ErrUploadUnsupported + case caps == nil && c.CapabilitiesKnown(): + return nil, ErrUploadNotAdvertised } // Uploads carry the file and the secret key, so they must never traverse @@ -110,11 +128,21 @@ func (c *Client) UploadFiles(paths []string) ([]UploadedFile, error) { switch resp.StatusCode { case http.StatusOK: - case http.StatusNotImplemented, http.StatusNotFound, http.StatusMethodNotAllowed: + case http.StatusNotImplemented: + // Advertised uploads and then refused them, which a mismatched + // deployment behind a load balancer can produce. The capability check + // above catches the ordinary case before a request is made. return nil, ErrUploadUnsupported case http.StatusUnauthorized: return nil, errkit.New("invalid token provided for interactsh server") default: + // 404 and 405 are deliberately not read as "uploads unsupported". The + // server answers 404 for an unknown correlation id, which is a session + // problem needing a re-register rather than a server flag, and a server + // predating /upload answers 200 from its catch-all rather than either + // status -- so mapping them here misdiagnosed the one case it caught and + // never caught the one it was aimed at. Reporting the status and body + // says something true whatever the cause. body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) return nil, fmt.Errorf("could not upload files (%s): %s", resp.Status, strings.TrimSpace(string(body))) } diff --git a/pkg/client/upload_test.go b/pkg/client/upload_test.go index a6100d83..982bf84b 100644 --- a/pkg/client/upload_test.go +++ b/pkg/client/upload_test.go @@ -38,6 +38,19 @@ func newUploadClient(t *testing.T, handler http.HandlerFunc, caps *server.Capabi if caps != nil { c.capabilities.Store(caps) } + // A registration happened, so whatever it said -- including nothing -- is + // authoritative. newResumedUploadClient covers the other case. + c.capabilitiesKnown.Store(true) + return c +} + +// newResumedUploadClient models a session resumed from -sf whose re-registration +// was refused because the session is still alive: no capabilities were ever +// received, so nothing is known about what the server offers. +func newResumedUploadClient(t *testing.T, handler http.HandlerFunc) *Client { + t.Helper() + c := newUploadClient(t, handler, nil) + c.capabilitiesKnown.Store(false) return c } @@ -48,6 +61,17 @@ func writeTempFile(t *testing.T, name string, content []byte) string { return p } +// errkit compares errors by message, so building ErrUploadNotAdvertised on an +// errkit sentinel made errors.Is match in both directions -- and the CLI checks the +// specific error first, so a server with -upload merely switched off was told to +// upgrade. The direction has to stay one-way. +func TestUploadSentinelsAreDirectional(t *testing.T) { + require.True(t, errors.Is(ErrUploadNotAdvertised, ErrUploadUnsupported), + "a server that advertised nothing cannot host files either") + require.False(t, errors.Is(ErrUploadUnsupported, ErrUploadNotAdvertised), + "a server that answered 501 is not a server that failed to advertise") +} + func TestUploadFiles(t *testing.T) { caps := &server.Capabilities{Upload: true, UploadMaxFileSize: 1024, UploadMaxFiles: 5, FTP: true} @@ -88,23 +112,95 @@ func TestUploadFiles(t *testing.T) { require.Equal(t, "/f/evil.dtd", files[0].HTTPPath) }) + // Capabilities are advertised here, so the request is actually made: a server + // that says it can host files and then refuses is the case 501 is left for. t.Run("501 reports unsupported", func(t *testing.T) { + var called atomic.Bool c := newUploadClient(t, func(w http.ResponseWriter, r *http.Request) { + called.Store(true) w.WriteHeader(http.StatusNotImplemented) - }, nil) + }, caps) _, err := c.UploadFiles([]string{writeTempFile(t, "a.dtd", []byte("x"))}) require.ErrorIs(t, err, ErrUploadUnsupported) + require.True(t, called.Load(), "the 501 path is only reachable by sending the request") }) - t.Run("404 and 405 report unsupported", func(t *testing.T) { - for _, code := range []int{http.StatusNotFound, http.StatusMethodNotAllowed} { - c := newUploadClient(t, func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(code) - }, nil) - _, err := c.UploadFiles([]string{writeTempFile(t, "a.dtd", []byte("x"))}) - require.ErrorIs(t, err, ErrUploadUnsupported, "status %d", code) - } + // 404 is what the server answers for an unknown correlation id -- a session + // problem needing a re-register, not a server missing a flag. Reporting it as + // "does not accept file uploads" sent the operator after the wrong remedy. + t.Run("404 surfaces the server's reason rather than claiming unsupported", func(t *testing.T) { + c := newUploadClient(t, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"error":"unknown correlation-id"}`)) + }, caps) + + _, err := c.UploadFiles([]string{writeTempFile(t, "a.dtd", []byte("x"))}) + require.Error(t, err) + require.Contains(t, err.Error(), "unknown correlation-id") + require.Contains(t, err.Error(), "404") + require.False(t, errors.Is(err, ErrUploadUnsupported), + "an unknown session must not be reported as a server without -upload") + }) + + t.Run("405 is surfaced rather than claiming unsupported", func(t *testing.T) { + c := newUploadClient(t, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusMethodNotAllowed) + }, caps) + + _, err := c.UploadFiles([]string{writeTempFile(t, "a.dtd", []byte("x"))}) + require.Error(t, err) + require.False(t, errors.Is(err, ErrUploadUnsupported)) + }) + + // A server advertising no capabilities predates file hosting. Its catch-all + // answers 200 with HTML, so without this the failure was an opaque + // "invalid character '<'" decode error. + t.Run("a server that advertised nothing is refused before any request", func(t *testing.T) { + var called atomic.Bool + c := newUploadClient(t, func(w http.ResponseWriter, r *http.Request) { + called.Store(true) + w.Header().Set("Content-Type", "text/html") + _, _ = w.Write([]byte("")) + }, nil) + + _, err := c.UploadFiles([]string{writeTempFile(t, "a.dtd", []byte("x"))}) + require.ErrorIs(t, err, ErrUploadNotAdvertised) + require.ErrorIs(t, err, ErrUploadUnsupported, + "callers asking only whether hosting is possible need no change") + require.False(t, called.Load(), "no point asking a server that cannot answer") + require.NotContains(t, err.Error(), "invalid character") + }) + + // Regression: a resumed session knows nothing about the server's capabilities, + // and treating that as "the server offers nothing" made every -sf resume + // refuse to upload, blaming a server that hosts files perfectly well. + t.Run("a resumed session attempts the upload rather than assuming", func(t *testing.T) { + var called atomic.Bool + c := newResumedUploadClient(t, func(w http.ResponseWriter, r *http.Request) { + called.Store(true) + _ = json.NewEncoder(w).Encode(&server.UploadResponse{ + Message: "upload successful", + Files: []server.UploadedFileResponse{{ + Name: "a.dtd", Size: 1, SHA256: "abc", HTTPPath: "/f/a.dtd", + }}, + }) + }) + + files, err := c.UploadFiles([]string{writeTempFile(t, "a.dtd", []byte("x"))}) + require.NoError(t, err, "an unknown capability set must not be read as unsupported") + require.True(t, called.Load(), "the request has to be made to find out") + require.Len(t, files, 1) + }) + + t.Run("a resumed session still reports a genuine 501", func(t *testing.T) { + c := newResumedUploadClient(t, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotImplemented) + }) + _, err := c.UploadFiles([]string{writeTempFile(t, "a.dtd", []byte("x"))}) + require.ErrorIs(t, err, ErrUploadUnsupported) + require.False(t, errors.Is(err, ErrUploadNotAdvertised), + "the server answered, so this is not version skew") }) t.Run("server error message is surfaced", func(t *testing.T) { From e51be9336690845fd8fffb96966bcdc644ef0773 Mon Sep 17 00:00:00 2001 From: nisay759 Date: Thu, 6 Aug 2026 16:25:00 +0200 Subject: [PATCH 18/20] docs: match the hosted-file examples to what the binaries print The README's client output and recorded-interaction examples predated two format changes on this branch. The ftp:// payload URL now carries the uploads directory, since hosted files live one level below the FTP root, and the elided body records "delivered of hosted" rather than a single count -- so a conditional fetch reads "0 of 144" and a ranged one reads the bytes the range carried. Both were captured from a real client against a real server built from this branch rather than edited by hand, including the blank lines the client emits between the dumped request and the response. A sentence now explains the two counts, since "144 of 144" is otherwise a puzzle. Co-Authored-By: Claude Opus 5 --- README.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9fcd7911..d1fc6818 100644 --- a/README.md +++ b/README.md @@ -701,7 +701,7 @@ interactsh-client -s https://hackwithautomation.com -t -file evil.dtd [INF] c6rj61aciaeutn2ae680cndmnioyyyyyn.hackwithautomation.com [INF] Hosting 1 file(s) for OOB Testing [INF] https://c6rj61aciaeutn2ae680xk4tqy8pqhwmi.hackwithautomation.com/f/evil.dtd -[INF] ftp://c6rj61aciaeutn2ae680xk4tqy8pqhwmi.hackwithautomation.com/c6rj61aciaeutn2ae680/evil.dtd +[INF] ftp://c6rj61aciaeutn2ae680xk4tqy8pqhwmi.hackwithautomation.com/.interactsh-user-uploads/c6rj61aciaeutn2ae680/evil.dtd ``` Files are served over HTTP(S), and over FTP(S) as well when `-ftp` is enabled. Responses are always @@ -725,6 +725,8 @@ Host: c6rj61aciaeutn2ae680xk4tqy8pqhwmi.hackwithautomation.com Accept: */* User-Agent: curl/8.18.0 + + ------------- HTTP Response ------------- @@ -734,9 +736,13 @@ Content-Type: application/octet-stream Content-Disposition: attachment; filename="evil.dtd" Content-Length: 144 -[body elided: 144 bytes of uploaded file "evil.dtd", sha256 0c1b960b076cdff8666f1f302dddd8f3ff0e6ed4b6c09002fbe6d1cdbb5d68f8] +[body elided: 144 of 144 bytes of uploaded file "evil.dtd", sha256 0c1b960b076cdff8666f1f302dddd8f3ff0e6ed4b6c09002fbe6d1cdbb5d68f8] ``` +The two counts are "delivered of hosted": a conditional fetch answered `304` records `0 of 144`, and a +ranged one records the bytes the range actually carried, so the record cannot claim a delivery that did +not happen. + Whatever second-stage callback the payload then triggers arrives as a further interaction on the same correlation ID, so both stages land in one client. From 3f58b9be6d39b13fcb42fa38519bcf15e23b52bb Mon Sep 17 00:00:00 2001 From: nisay759 Date: Thu, 6 Aug 2026 16:51:20 +0200 Subject: [PATCH 19/20] style: gofmt the two files this branch adds fields to Whitespace only -- git diff -w is empty. Both files were already gofmt-unclean before this branch, but adding the upload options to CLIServerOptions and UploadedFile to the storage types widens those struct blocks, so more of the surrounding pre-existing fields fall out of alignment: server_options.go went from 18 gofmt-changed lines to 30. Formatting them keeps the files this branch edits clean without dragging in realignment of files it never touched. Left alone deliberately, since they are unrelated to this change and would only make the diff harder to review: pkg/server/acme/acme_certbot.go, pkg/server/acme/cert_reloader.go, pkg/server/http_server_test.go, pkg/server/responder_server.go, pkg/storage/storage_redis_test.go and pkg/storage/storagedb_test.go. Co-Authored-By: Claude Opus 5 --- pkg/options/server_options.go | 30 +++++++++++++++--------------- pkg/storage/types.go | 2 +- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/pkg/options/server_options.go b/pkg/options/server_options.go index 110b9f18..21b1fdad 100644 --- a/pkg/options/server_options.go +++ b/pkg/options/server_options.go @@ -56,21 +56,21 @@ type CLIServerOptions struct { // RedisURL, when set, switches the server to a Redis-backed storage // backend so multiple instances can share state behind a load balancer. // Disk storage flags are ignored when RedisURL is set. - RedisURL string - RedisKeyPrefix string - EnablePprof bool - EnableMetrics bool - Verbose bool - DisableUpdateCheck bool - NoVersionHeader bool - HeaderServer string - DefaultHTTPResponseFile string - Upload bool - UploadDirectory string - UploadMaxFileSize goflags.Size - UploadMaxFiles int - UploadMaxTotalSize goflags.Size - UploadTTL time.Duration + RedisURL string + RedisKeyPrefix string + EnablePprof bool + EnableMetrics bool + Verbose bool + DisableUpdateCheck bool + NoVersionHeader bool + HeaderServer string + DefaultHTTPResponseFile string + Upload bool + UploadDirectory string + UploadMaxFileSize goflags.Size + UploadMaxFiles int + UploadMaxTotalSize goflags.Size + UploadTTL time.Duration } func (cliServerOptions *CLIServerOptions) AsServerOptions() *server.Options { diff --git a/pkg/storage/types.go b/pkg/storage/types.go index 13f1e485..0b8084bc 100644 --- a/pkg/storage/types.go +++ b/pkg/storage/types.go @@ -36,7 +36,7 @@ type CorrelationData struct { // AESKey is the AES encryption key in encrypted format. AESKeyEncrypted string `json:"aes-key"` // decrypted AES key for signing - AESKey []byte `json:"-"` + AESKey []byte `json:"-"` ReadOffsets map[string]int `json:"-"` LastSeen map[string]time.Time `json:"-"` // Files is metadata for files uploaded against this correlation-id. From fb9f1e52d10f7cd21b8fda2a4a7682d98892de70 Mon Sep 17 00:00:00 2001 From: nisay759 Date: Thu, 6 Aug 2026 17:21:01 +0200 Subject: [PATCH 20/20] examples: show the file-hosting capability in the client library example The library example only demonstrated polling for interactions, so nothing showed a consumer how to reach the feature this branch adds -- or that it has to negotiate for it. It now uploads a file, prints the http:// and ftp:// URLs a target would fetch, fetches one itself, and shows both fetches arriving in the poll callback, which is the whole point of hosting: the retrieval is the evidence. Hosting is optional, and the servers DefaultOptions points at deliberately do not offer it, so the example has to degrade rather than fail. It checks Capabilities() first and handles both refusals from UploadFiles separately, since ErrUploadNotAdvertised means "upgrade the server" while ErrUploadUnsupported means "start it with -upload" -- different remedies, and a consumer that conflates them tells its user the wrong thing. An unknown capability set, as a resumed session has, falls through to attempting the upload rather than assuming either way. The client variable is renamed from client to c so the package remains reachable for those exported errors; it was previously shadowed after assignment. README's library section now says hosting is a negotiated capability and names the four pieces of API involved. Co-Authored-By: Claude Opus 5 --- README.md | 10 ++++- examples/client.go | 96 +++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 100 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index d1fc6818..1d2bf07f 100644 --- a/README.md +++ b/README.md @@ -1032,7 +1032,15 @@ sudo interactsh-server -responder -d localhost ### Use as library -The [examples](examples/) uses interactsh client library to get external interactions for a generated URL by making a http request to the URL. +The [examples](examples/) use the interactsh client library to get external interactions for a generated +URL by making an http request to the URL, and to host a file against the same session for second-stage +verification. + +File hosting is an optional server capability, so a library consumer negotiates rather than assumes: +`Capabilities()` reports what the server advertised at registration, `UploadFiles` returns +`ErrUploadUnsupported` when a server cannot host files and `ErrUploadNotAdvertised` when it predates the +feature, and `FileURL`/`FTPFileURL` compose the URLs a target should fetch. The public `oast.*` servers +do not offer hosting, so the example skips it rather than failing. ### Nuclei - OAST diff --git a/examples/client.go b/examples/client.go index 6f261e9f..e64be5d9 100644 --- a/examples/client.go +++ b/examples/client.go @@ -1,8 +1,10 @@ package main import ( + "errors" "fmt" "net/http" + "os" "time" "github.com/projectdiscovery/interactsh/pkg/client" @@ -10,28 +12,30 @@ import ( ) func main() { - client, err := client.New(client.DefaultOptions) + // Named c rather than client so the package stays reachable for its + // exported errors below. + c, err := client.New(client.DefaultOptions) if err != nil { panic(err) } defer func() { - if err := client.Close(); err != nil { + if err := c.Close(); err != nil { panic(err) } }() - if err := client.StartPolling(time.Duration(1*time.Second), func(interaction *server.Interaction) { + if err := c.StartPolling(time.Duration(1*time.Second), func(interaction *server.Interaction) { fmt.Printf("Got Interaction: %v => %v\n", interaction.Protocol, interaction.FullId) }); err != nil { panic(err) } defer func() { - if err := client.StopPolling(); err != nil { + if err := c.StopPolling(); err != nil { panic(err) } }() - URL := client.URL() + URL := c.URL() resp, err := http.Get("https://" + URL) if err != nil { @@ -42,5 +46,87 @@ func main() { } fmt.Printf("Got URL: %v => %v\n", URL, resp) + + // Second stage: host a file against this session, if the server offers it. + hostFile(c, URL) + time.Sleep(1 * time.Second) } + +// hostFile uploads a file and fetches it back, which is how a second-stage +// payload is verified: the target retrieves the hosted file, and that retrieval +// arrives as an interaction of its own. +// +// Hosting is optional, so a caller has to handle its absence. The server +// advertises it at registration, and the public oast.* servers deliberately do +// not offer it, so this returns quietly rather than failing. +func hostFile(c *client.Client, payloadHost string) { + caps := c.Capabilities() + switch { + case caps == nil && c.CapabilitiesKnown(): + // The server answered and said nothing about capabilities, so it + // predates file hosting. + fmt.Println("Hosting: server predates file hosting, skipping") + return + case caps != nil && !caps.Upload: + fmt.Println("Hosting: server does not offer file hosting, skipping") + return + } + // caps == nil with nothing known -- a resumed session, say -- falls through: + // the only way to find out is to ask, and UploadFiles reports what it learns. + + path, err := writeTempFile(``) + if err != nil { + fmt.Printf("Hosting: %v\n", err) + return + } + defer func() { _ = os.Remove(path) }() + + files, err := c.UploadFiles([]string{path}) + if err != nil { + // Distinguishable so a caller can tell "cannot host" from "the request + // failed", and act on the right one. + switch { + case errors.Is(err, client.ErrUploadNotAdvertised): + fmt.Println("Hosting: server predates file hosting, skipping") + case errors.Is(err, client.ErrUploadUnsupported): + fmt.Println("Hosting: server does not offer file hosting, skipping") + default: + fmt.Printf("Hosting: upload failed: %v\n", err) + } + return + } + + for _, file := range files { + // Any host from URL() works: the server reads only its correlation ID. + fileURL := c.FileURL(payloadHost, file) + fmt.Printf("Hosting %s (%d bytes, sha256 %s) => %s\n", file.Name, file.Size, file.SHA256, fileURL) + if caps != nil && caps.FTP { + fmt.Printf("Hosting %s over FTP => %s\n", file.Name, c.FTPFileURL(payloadHost, file)) + } + + // Stand in for the target fetching it; the fetch is recorded as an + // interaction against this session and arrives in the poll callback. + resp, err := http.Get(fileURL) + if err != nil { + fmt.Printf("Hosting: could not fetch %s: %v\n", fileURL, err) + continue + } + if err := resp.Body.Close(); err != nil { + panic(err) + } + fmt.Printf("Fetched %s => %v\n", fileURL, resp.Status) + } +} + +func writeTempFile(content string) (string, error) { + f, err := os.CreateTemp("", "interactsh-example-*.dtd") + if err != nil { + return "", err + } + defer func() { _ = f.Close() }() + if _, err := f.WriteString(content); err != nil { + return "", err + } + return f.Name(), nil +}