diff --git a/ROADMAP.md b/ROADMAP.md index 112f656..4cfe77b 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -8,10 +8,11 @@ | Mentions | ✅ | ✅ | | Images and GIFs | ✅ | ✅ | | Video | ✅ | ⚠️ Uploads, but GroupMe can't play it back | +| Voice notes | ✅ | ✅ | | Files | ✅ Groups only | ✅ Groups only | | Media captions | ✅ | ✅ | | Locations | ✅ | ✅ | -| Direct media | ✅ Images, videos, group files and avatars | N/A | +| Direct media | ✅ Images, videos, voice notes, group files and avatars | N/A | | Emoji reactions | ⚠️ Push events, reconciled on reconnect | ✅ | | Custom reaction icons | ❌ | ❌ | | Polls | ✅ Active polls; reminders and results as text | ⚠️ Voting only; no creation or vote withdrawal | diff --git a/go.mod b/go.mod index 74cc3eb..7fed247 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.26.0 toolchain go1.27.1 require ( - github.com/beeper/groupme-lib v0.2.1-0.20261027164818-a385d1c65167 + github.com/beeper/groupme-lib v0.2.1-0.20261027180653-c4a4c74d6dd4 github.com/coder/websocket v1.8.15 github.com/google/uuid v1.2.0 github.com/mattn/go-sqlite3 v1.14.52 diff --git a/go.sum b/go.sum index 04007c4..2c0462d 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,8 @@ filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo= filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc= github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= -github.com/beeper/groupme-lib v0.2.1-0.20261027164818-a385d1c65167 h1:OjaO3JFqQp2tuxDLvFzTk6MVNXu/7AaSgCaEMHaVfLU= -github.com/beeper/groupme-lib v0.2.1-0.20261027164818-a385d1c65167/go.mod h1:FYHpsnrrdRvw0kXIiflXpV2NAnlPHPHGgzIf/wSy/mM= +github.com/beeper/groupme-lib v0.2.1-0.20261027180653-c4a4c74d6dd4 h1:vIGN8y93kyKNOw5VF6B49EhQDSzKunHk71gn2Kf1lt0= +github.com/beeper/groupme-lib v0.2.1-0.20261027180653-c4a4c74d6dd4/go.mod h1:FYHpsnrrdRvw0kXIiflXpV2NAnlPHPHGgzIf/wSy/mM= github.com/coder/websocket v1.8.15 h1:6B2JPeOGlpff2Uz6vOEH1Vzpi0iUz20A+lPVhPHtNUA= github.com/coder/websocket v1.8.15/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg= github.com/coreos/go-systemd/v22 v22.7.0 h1:LAEzFkke61DFROc7zNLX/WA2i5J8gYqe0rSj9KI28KA= diff --git a/pkg/connector/capabilities.go b/pkg/connector/capabilities.go index 0d77072..73afb37 100644 --- a/pkg/connector/capabilities.go +++ b/pkg/connector/capabilities.go @@ -57,13 +57,18 @@ var videoMimes = map[string]event.CapabilitySupportLevel{ "video/mp4": event.CapLevelFullySupported, } +// Beeper clients transcode voice recordings to a listed type before sending. +var voiceMimes = map[string]event.CapabilitySupportLevel{ + "audio/mp4": event.CapLevelFullySupported, +} + var fileMimes = map[string]event.CapabilitySupportLevel{ "application/*": event.CapLevelFullySupported, "text/*": event.CapLevelFullySupported, } var roomCaps = &event.RoomFeatures{ - ID: "fi.mau.groupme.capabilities.2026_10_26.4", + ID: "fi.mau.groupme.capabilities.2026_10_26.5", Formatting: event.FormattingFeatureMap{ event.FmtUserLink: event.CapLevelFullySupported, }, @@ -78,6 +83,11 @@ var roomCaps = &event.RoomFeatures{ Caption: event.CapLevelFullySupported, MaxSize: MaxFileSize, }, + event.CapMsgVoice: { + MimeTypes: voiceMimes, + Caption: event.CapLevelDropped, + MaxSize: MaxFileSize, + }, event.MsgFile: { MimeTypes: fileMimes, Caption: event.CapLevelFullySupported, diff --git a/pkg/connector/directmedia.go b/pkg/connector/directmedia.go index b9bed9a..06cf50d 100644 --- a/pkg/connector/directmedia.go +++ b/pkg/connector/directmedia.go @@ -27,7 +27,7 @@ func (gc *GMConnector) SetUseDirectMedia() { // Credentials stay in login metadata; the framework signs and serves media IDs. // // 1i: image URL -// 1v: login ID, video URL +// 1v: login ID, video or voice note URL // 1f: login ID, group ID, file ID func (gc *GMConnector) directMediaURI(ctx context.Context, fields ...string) (id.ContentURIString, error) { mediaID := networkid.MediaID(strings.Join(fields, "\x00")) @@ -111,6 +111,9 @@ func (gc *GMClient) directMediaContent(ctx context.Context, msg *groupme.Message case groupme.Video: fields = []string{"1v", string(gc.UserLogin.ID), att.URL} content.MsgType, content.Body = event.MsgVideo, "video" + case groupme.Audio: + fields = []string{"1v", string(gc.UserLogin.ID), att.URL} + content.MsgType, content.Body = event.MsgAudio, "voice note" case groupme.File: fields = []string{"1f", string(gc.UserLogin.ID), string(msg.GroupID), att.FileID} meta, err := groupmeext.GetFileMetadata(ctx, msg.GroupID, att.FileID, gc.Meta.Token) diff --git a/pkg/connector/handlegroupme.go b/pkg/connector/handlegroupme.go index b6aab63..c8d618f 100644 --- a/pkg/connector/handlegroupme.go +++ b/pkg/connector/handlegroupme.go @@ -148,7 +148,7 @@ func (gc *GMClient) convertGroupMeMessage(ctx context.Context, portal *bridgev2. partID := networkid.PartID(fmt.Sprintf("attachment-%d", i)) var content *event.MessageEventContent switch att.Type { - case groupme.Image, groupme.Video, groupme.File: + case groupme.Image, groupme.Video, groupme.Audio, groupme.File: if att.Type == groupme.File && msg.GroupID == "" { log.Warn().Msg("Ignoring GroupMe file attachment outside a group") continue @@ -170,7 +170,7 @@ func (gc *GMClient) convertGroupMeMessage(ctx context.Context, portal *bridgev2. switch att.Type { case groupme.Image: data, mime, err = groupmeext.DownloadImage(ctx, att.URL) - case groupme.Video: + case groupme.Video, groupme.Audio: data, mime, err = groupmeext.DownloadVideo(ctx, att.URL, gc.Meta.Token) case groupme.File: data, filename, mime, err = groupmeext.DownloadFile(ctx, msg.GroupID, att.FileID, gc.Meta.Token) @@ -232,6 +232,9 @@ func (gc *GMClient) convertGroupMeMessage(ctx context.Context, portal *bridgev2. // Mentions and custom emoji retain their plaintext fallback. continue } + if att.Type == groupme.Audio { + addVoiceNoteInfo(content, att) + } cm.Parts = append(cm.Parts, &bridgev2.ConvertedMessagePart{ ID: partID, diff --git a/pkg/connector/handlematrix.go b/pkg/connector/handlematrix.go index 93f4ec0..988e8b4 100644 --- a/pkg/connector/handlematrix.go +++ b/pkg/connector/handlematrix.go @@ -17,6 +17,7 @@ package connector import ( + "cmp" "context" "errors" "fmt" @@ -122,8 +123,11 @@ func (gc *GMClient) HandleMatrixMessage(ctx context.Context, msg *bridgev2.Matri return nil, fmt.Errorf("invalid GroupMe portal") } text := content.Body - if content.MsgType == event.MsgImage || content.MsgType == event.MsgVideo || content.MsgType == event.MsgFile { + switch content.MsgType { + case event.MsgImage, event.MsgVideo, event.MsgFile: text = content.GetCaption() + case event.MsgAudio: + text = "" } fmtCtx := format.NewContext(ctx) if text != "" && content.Format == event.FormatHTML && content.FormattedBody != "" { @@ -148,7 +152,7 @@ func (gc *GMClient) HandleMatrixMessage(ctx context.Context, msg *bridgev2.Matri switch content.MsgType { case event.MsgText, event.MsgNotice, event.MsgEmote: - case event.MsgImage, event.MsgVideo, event.MsgFile: + case event.MsgImage, event.MsgVideo, event.MsgAudio, event.MsgFile: attachment, err := gc.uploadMatrixMedia(ctx, content, portalType, gmid) if err != nil { return nil, err @@ -213,7 +217,7 @@ func (gc *GMClient) HandleMatrixMessage(ctx context.Context, msg *bridgev2.Matri } // uploadMatrixMedia shares the Matrix download and byte limit checks. GroupMe -// has separate image, video-session, and group-file upload APIs. +// has separate image, media-session, and group-file upload APIs. func (gc *GMClient) uploadMatrixMedia(ctx context.Context, content *event.MessageEventContent, portalType PortalType, gmid groupme.ID) (*groupme.Attachment, error) { data, err := gc.Main.br.Bot.DownloadMedia(ctx, content.URL, content.File) if err != nil { @@ -237,21 +241,23 @@ func (gc *GMClient) uploadMatrixMedia(ctx context.Context, content *event.Messag return nil, wrapGroupMeError(err, "failed to upload media").WithIsCertain(true) } return &groupme.Attachment{Type: groupme.Image, URL: url}, nil - case event.MsgVideo: - if mimeType == "" { - mimeType = "video/mp4" - } + case event.MsgVideo, event.MsgAudio: + var extension string + mimeType, extension = mediaUploadType(content, mimeType) var groupID, recipientID string if portalType == PortalTypeDM { recipientID = string(gmid) } else { groupID = string(gmid) } - url, preview, err := groupmeext.UploadVideo(ctx, gc.Meta.Token, gc.Meta.GMID, groupID, recipientID, data, videoExtension(content, mimeType), mimeType) + uploaded, err := groupmeext.UploadMedia(ctx, gc.Meta.Token, gc.Meta.GMID, groupID, recipientID, data, extension, mimeType) if err != nil { return nil, wrapGroupMeError(err, "failed to upload media").WithIsCertain(true) } - return &groupme.Attachment{Type: groupme.Video, URL: url, VideoPreviewURL: preview}, nil + if content.MsgType == event.MsgAudio { + return voiceNoteAttachment(content, uploaded), nil + } + return &groupme.Attachment{Type: groupme.Video, URL: uploaded.RenderURL, VideoPreviewURL: uploaded.ThumbnailURL}, nil case event.MsgFile: fileID, err := groupmeext.UploadFile(ctx, gmid, gc.Meta.Token, content.GetFileName(), data, mimeType) if err != nil { @@ -263,15 +269,24 @@ func (gc *GMClient) uploadMatrixMedia(ctx context.Context, content *event.Messag } } -// The video upload session requires an extension before it receives any bytes. -func videoExtension(content *event.MessageEventContent, mimeType string) string { +// The media upload session requires an extension before it receives any bytes. +func mediaUploadType(content *event.MessageEventContent, mimeType string) (string, string) { + if content.MsgType == event.MsgAudio { + mimeType = cmp.Or(mimeType, "audio/mp4") + } else { + mimeType = cmp.Or(mimeType, "video/mp4") + } if ext := strings.TrimPrefix(filepath.Ext(content.GetFileName()), "."); ext != "" { - return strings.ToLower(ext) + return mimeType, strings.ToLower(ext) + } + if content.MsgType == event.MsgAudio { + // mime lists audio/mp4 extensions alphabetically, so .f4a would come first. + return mimeType, "m4a" } if exts, err := mime.ExtensionsByType(mimeType); err == nil && len(exts) > 0 { - return strings.ToLower(strings.TrimPrefix(exts[0], ".")) + return mimeType, strings.ToLower(strings.TrimPrefix(exts[0], ".")) } - return "mp4" + return mimeType, "mp4" } // GroupMe location attachments carry latitude, longitude, and a name. diff --git a/pkg/connector/mediainfo.go b/pkg/connector/mediainfo.go index 1e3444c..f53ddbe 100644 --- a/pkg/connector/mediainfo.go +++ b/pkg/connector/mediainfo.go @@ -17,6 +17,8 @@ package connector import ( + "encoding/json" + "math" "mime" "net/url" "path" @@ -29,6 +31,7 @@ import ( "maunium.net/go/mautrix/event" "github.com/beeper/groupme-lib" + "github.com/beeper/groupme/pkg/groupmeext" ) // GroupMe image URLs name the file after its dimensions and format, e.g. @@ -81,5 +84,64 @@ func visibleText(msg *groupme.Message) string { }) { return "" } + if strings.HasPrefix(text, "⚠️You received a voice note") && slices.ContainsFunc(msg.Attachments, func(att *groupme.Attachment) bool { + return att != nil && att.Type == groupme.Audio + }) { + return "" + } return text } + +// Matrix voice waveforms range from 0 to 1024; GroupMe peaks from 0 to 1. +const waveformScale = 1024 + +func addVoiceNoteInfo(content *event.MessageEventContent, att *groupme.Attachment) { + if content.Info == nil { + content.Info = &event.FileInfo{} + } + content.Info.Duration = int(att.Duration * 1000) + content.MSC3245Voice = &event.MSC3245Voice{} + content.MSC1767Audio = &event.MSC1767Audio{Duration: content.Info.Duration} + for _, peak := range parsePeaks(att.Peaks) { + content.MSC1767Audio.Waveform = append(content.MSC1767Audio.Waveform, int(math.Round(min(max(peak, 0), 1)*waveformScale))) + } +} + +func parsePeaks(raw json.RawMessage) []float64 { + var encoded string + if json.Unmarshal(raw, &encoded) == nil { + // Live DM events send comma-separated peaks without array brackets. + encoded = strings.TrimSpace(encoded) + if !strings.HasPrefix(encoded, "[") { + encoded = "[" + encoded + "]" + } + raw = json.RawMessage(encoded) + } + var peaks []float64 + _ = json.Unmarshal(raw, &peaks) + return peaks +} + +func voiceNoteAttachment(content *event.MessageEventContent, uploaded *groupmeext.UploadedMedia) *groupme.Attachment { + att := &groupme.Attachment{Type: groupme.Audio, URL: uploaded.RenderURL, TranscriptURL: uploaded.TranscriptURL} + durationMS := 0 + if content.MSC1767Audio != nil { + durationMS = content.MSC1767Audio.Duration + if len(content.MSC1767Audio.Waveform) > 0 { + peaks := make([]float64, len(content.MSC1767Audio.Waveform)) + for i, sample := range content.MSC1767Audio.Waveform { + peaks[i] = min(max(float64(sample)/waveformScale, 0), 1) + } + encoded, _ := json.Marshal(peaks) + att.Peaks, _ = json.Marshal(string(encoded)) + } + } + if durationMS == 0 && content.Info != nil { + durationMS = content.Info.Duration + } + // GroupMe's clients send whole seconds. + if durationMS > 0 { + att.Duration = max(1, math.Round(float64(durationMS)/1000)) + } + return att +} diff --git a/pkg/connector/mediainfo_test.go b/pkg/connector/mediainfo_test.go new file mode 100644 index 0000000..a7e1573 --- /dev/null +++ b/pkg/connector/mediainfo_test.go @@ -0,0 +1,32 @@ +package connector + +import ( + "encoding/json" + "slices" + "testing" + + "maunium.net/go/mautrix/event" + + "github.com/beeper/groupme-lib" +) + +func TestVoiceNoteNativePeakFormats(t *testing.T) { + for name, peaks := range map[string]string{ + "array": `[0,0.125,0.5,1,0]`, + "group string": `"[0,0.125,0.5,1,0]"`, + "live DM string": `"0.0,0.125,0.5,1.0,0.0"`, + } { + t.Run(name, func(t *testing.T) { + content := &event.MessageEventContent{} + addVoiceNoteInfo(content, &groupme.Attachment{ + Type: groupme.Audio, Duration: 3, Peaks: json.RawMessage(peaks), + }) + if content.MSC3245Voice == nil || content.Info.Duration != 3000 || content.MSC1767Audio.Duration != 3000 { + t.Fatal("voice flag or duration missing") + } + if !slices.Equal(content.MSC1767Audio.Waveform, []int{0, 128, 512, 1024, 0}) { + t.Fatalf("unexpected waveform: %v", content.MSC1767Audio.Waveform) + } + }) + } +} diff --git a/pkg/groupmeext/message.go b/pkg/groupmeext/message.go index 40967ed..8c0cf0a 100644 --- a/pkg/groupmeext/message.go +++ b/pkg/groupmeext/message.go @@ -18,7 +18,8 @@ func DownloadImage(ctx context.Context, imageURL string) ([]byte, string, error) return downloadMediaURL(ctx, imageURL, "") } -// DownloadVideo uses the native token-cookie authentication contract. +// DownloadVideo uses the native token-cookie authentication contract, which +// voice notes share. func DownloadVideo(ctx context.Context, videoURL, token string) ([]byte, string, error) { return downloadMediaURL(ctx, videoURL, token) } @@ -113,7 +114,7 @@ func GetFileMetadata(ctx context.Context, groupID groupme.ID, fileID, token stri return &meta[0].FileData, nil } -// Video sessions require a group ID or recipient ID, plus size and extension. +// Upload sessions require a group ID or recipient ID, plus size and extension. type createUploadRequest struct { FileSize int64 `json:"FileSize"` SenderId string `json:"SenderId"` @@ -122,17 +123,19 @@ type createUploadRequest struct { RecipientId string `json:"recipientId,omitempty"` } -// UploadURL is a temporary signed upload destination. RenderURL and ThumbnailURL -// are attachment URLs; no expiry parameter was observed in those URLs. -type createUploadResponse struct { - UploadURL string `json:"uploadUrl"` - RenderURL string `json:"renderUrl"` - ThumbnailURL string `json:"thumbnailUrl"` +// UploadURL is a temporary signed upload destination. The other URLs are +// attachment URLs; no expiry parameter was observed in them. +type UploadedMedia struct { + UploadURL string `json:"uploadUrl"` + RenderURL string `json:"renderUrl"` + ThumbnailURL string `json:"thumbnailUrl"` + TranscriptURL string `json:"transcriptUrl"` } -// UploadVideo creates a session, then PUTs bytes to its signed URL without -// GroupMe credentials. Exactly one of groupID and recipientID must be set. -func UploadVideo(ctx context.Context, token, senderID, groupID, recipientID string, data []byte, extension, mimeType string) (renderURL, thumbnailURL string, err error) { +// UploadMedia uploads a video or voice note: it creates a session, then PUTs +// bytes to its signed URL without GroupMe credentials. Exactly one of groupID +// and recipientID must be set. +func UploadMedia(ctx context.Context, token, senderID, groupID, recipientID string, data []byte, extension, mimeType string) (*UploadedMedia, error) { reqBody, err := json.Marshal(createUploadRequest{ FileSize: int64(len(data)), SenderId: senderID, @@ -141,33 +144,33 @@ func UploadVideo(ctx context.Context, token, senderID, groupID, recipientID stri RecipientId: recipientID, }) if err != nil { - return "", "", fmt.Errorf("failed to build upload session request: %w", err) + return nil, fmt.Errorf("failed to build upload session request: %w", err) } sessionReq, err := http.NewRequestWithContext(ctx, http.MethodPost, "https://m.groupme.com/uploads", bytes.NewReader(reqBody)) if err != nil { - return "", "", fmt.Errorf("failed to build upload session request: %w", err) + return nil, fmt.Errorf("failed to build upload session request: %w", err) } sessionReq.Header.Set("X-Access-Token", token) sessionReq.Header.Set("Content-Type", "application/json") sessionResp, err := mediaRequest(sessionReq) if err != nil { - return "", "", fmt.Errorf("failed to create upload session: %w", err) + return nil, fmt.Errorf("failed to create upload session: %w", err) } defer sessionResp.Body.Close() - var session createUploadResponse + var session UploadedMedia if err := json.NewDecoder(io.LimitReader(sessionResp.Body, 1024*1024)).Decode(&session); err != nil { - return "", "", fmt.Errorf("failed to decode upload session response: %w", err) + return nil, fmt.Errorf("failed to decode upload session response: %w", err) } if session.UploadURL == "" || session.RenderURL == "" { - return "", "", fmt.Errorf("GroupMe did not return an upload URL for the video session") + return nil, fmt.Errorf("GroupMe did not return an upload URL for the media session") } putReq, err := http.NewRequestWithContext(ctx, http.MethodPut, session.UploadURL, bytes.NewReader(data)) if err != nil { - return "", "", fmt.Errorf("invalid GroupMe video upload URL") + return nil, fmt.Errorf("invalid GroupMe media upload URL") } putReq.Header.Set("Content-Type", mimeType) // Required by Azure Blob Storage for a PUT that creates a new blob @@ -177,11 +180,11 @@ func UploadVideo(ctx context.Context, token, senderID, groupID, recipientID stri putResp, err := mediaRequest(putReq) if err != nil { - return "", "", fmt.Errorf("failed to upload video bytes: %w", err) + return nil, fmt.Errorf("failed to upload media bytes: %w", err) } defer putResp.Body.Close() - return session.RenderURL, session.ThumbnailURL, nil + return &session, nil } // createFileResponse is file.groupme.com's response to starting a file @@ -205,7 +208,7 @@ type fileUploadStatus struct { // UploadFile uploads an outgoing file (GroupMe's group file-sharing // feature -- see DownloadFile's doc comment) and returns its file_id, for // use as a "file" attachment's file_id. Reverse-engineered live the same -// way as UploadVideo. +// way as UploadMedia. // // Unlike video, this doesn't need a separate SAS-signed upload step -- // file.groupme.com is GroupMe's own file-service (confirmed via its