feat(docs): Normalize documentation and enrich text rendering

README.md

@@ -115,12 +115,14 @@ apple-docs types view MXHangDiagnostic --technology MetricKit
 
 The terminal output includes available information such as:
 
-- Summary and declaration
-- Platform availability and deprecation
+- Summary and declarations in the languages supplied by Apple
+- Platform availability, deprecation, obsoleted versions, beta status, and unavailability
 - Inheritance and protocol conformances
 - Documented members and related APIs
 - Canonical Apple Developer URL
 
+Text rendering also supports sparse article and collection pages, resolves reference links, and strips remote terminal control characters. It uses normalized documentation content, which does not cover every upstream DocC field. Use `--json` when you need the original document.
+
 Every `types` command requires `--technology`. The CLI does not persist a selected framework. Nested symbols accept either dotted Swift spelling or slash-separated DocC paths:
 
 ```bash

Sources/CLI/client/AppleDocumentationClient+DTO.swift

@@ -8,6 +8,32 @@ struct TypeDocumentationPageDTO: Decodable, Sendable {
     let seeAlsoSections: [DocumentationReferenceSectionDTO]?
     let topicSections: [DocumentationReferenceSectionDTO]?
     let variants: [DocumentationVariantDTO]?
+    let kind: String?
+
+    private enum CodingKeys: CodingKey {
+        case abstract, deprecationSummary, metadata, primaryContentSections, references
+        case relationshipsSections, seeAlsoSections, topicSections, variants, kind
+    }
+
+    init(from decoder: any Decoder) throws {
+        let container = try decoder.container(keyedBy: CodingKeys.self)
+        metadata = try container.decode(DocumentationMetadataDTO.self, forKey: .metadata)
+        abstract = try container.decodeIfPresent([DocumentationTextDTO].self, forKey: .abstract) ?? []
+        deprecationSummary = try container.decodeIfPresent([DocumentationBlockDTO].self, forKey: .deprecationSummary)
+        primaryContentSections =
+            try container.decodeIfPresent(
+                [DocumentationContentSectionDTO].self, forKey: .primaryContentSections
+            ) ?? []
+        references = try container.decodeIfPresent([String: DocumentationReferenceDTO].self, forKey: .references) ?? [:]
+        relationshipsSections = try container.decodeIfPresent(
+            [DocumentationReferenceSectionDTO].self, forKey: .relationshipsSections
+        )
+        seeAlsoSections = try container.decodeIfPresent(
+            [DocumentationReferenceSectionDTO].self, forKey: .seeAlsoSections)
+        topicSections = try container.decodeIfPresent([DocumentationReferenceSectionDTO].self, forKey: .topicSections)
+        variants = try container.decodeIfPresent([DocumentationVariantDTO].self, forKey: .variants)
+        kind = try container.decodeIfPresent(String.self, forKey: .kind)
+    }
 }
 
 struct DocumentationVariantDTO: Decodable, Sendable {
@@ -18,6 +44,7 @@ struct DocumentationTextDTO: Decodable, Sendable {
     let code: String?
     let identifier: String?
     let inlineContent: [DocumentationTextDTO]?
+    let overridingTitle: String?
     let text: String?
 }
 
@@ -111,13 +138,15 @@ struct DocumentationMetadataDTO: Decodable, Sendable {
     let modules: [DocumentationModule]
     let platforms: [DocumentationPlatform]
     let roleHeading: String
+    let role: String?
     let symbolKind: String?
     let title: String
 
     private enum CodingKeys: CodingKey {
         case modules
         case platforms
         case roleHeading
+        case role
         case symbolKind
         case title
     }
@@ -126,7 +155,8 @@ struct DocumentationMetadataDTO: Decodable, Sendable {
         let container = try decoder.container(keyedBy: CodingKeys.self)
         modules = try container.decodeIfPresent([DocumentationModule].self, forKey: .modules) ?? []
         platforms = try container.decodeIfPresent([DocumentationPlatform].self, forKey: .platforms) ?? []
-        roleHeading = try container.decode(String.self, forKey: .roleHeading)
+        role = try container.decodeIfPresent(String.self, forKey: .role)
+        roleHeading = try container.decodeIfPresent(String.self, forKey: .roleHeading) ?? ""
         symbolKind = try container.decodeIfPresent(String.self, forKey: .symbolKind)
         title = try container.decode(String.self, forKey: .title)
     }
@@ -140,4 +170,7 @@ struct DocumentationPlatform: Decodable, Sendable {
     let deprecatedAt: String?
     let introducedAt: String?
     let name: String
+    let obsoletedAt: String?
+    let beta: Bool?
+    let unavailable: Bool?
 }

Sources/CLI/client/AppleDocumentationClient.swift

@@ -46,9 +46,7 @@ struct DefaultAppleDocumentationClient<Dependencies: DefaultAppleDocumentationCl
         ]
         logger.debug("Fetching type documentation", metadata: metadata)
         do {
-            let document = TypeDocumentationDocument(
-                data: try await fetchData(from: typeURL(name: name, technology: technology))
-            )
+            let document = try await fetchDocument(at: typeDestination(name: name, technology: technology))
             logger.info("Fetched type documentation", metadata: metadata)
             return document
         } catch Error.httpStatus(404) {
@@ -63,9 +61,7 @@ struct DefaultAppleDocumentationClient<Dependencies: DefaultAppleDocumentationCl
             if slug.caseInsensitiveCompare(technology) != .orderedSame {
                 logger.debug("Retrying type with canonical technology", metadata: ["slug": .string(slug)])
                 do {
-                    let document = TypeDocumentationDocument(
-                        data: try await fetchData(from: typeURL(name: name, technology: slug))
-                    )
+                    let document = try await fetchDocument(at: typeDestination(name: name, technology: slug))
                     logger.info("Fetched type documentation", metadata: metadata)
                     return document
                 } catch Error.httpStatus(404) {
@@ -130,21 +126,38 @@ struct DefaultAppleDocumentationClient<Dependencies: DefaultAppleDocumentationCl
         return types
     }
 
-    func fetchDocumentationPage(path: String) async throws -> TechnologyDocumentationPageDTO {
+    func fetchDocument(at destination: DocumentationDestination) async throws -> TypeDocumentationDocument {
+        TypeDocumentationDocument(
+            data: try await fetchDocumentationData(path: destination.path), destination: destination
+        )
+    }
+
+    func fetchRootDocument(technology: String) async throws -> TypeDocumentationDocument {
+        try await fetchDocumentationRoot(technology: technology).document
+    }
+
+    private func fetchDocumentationData(path: String) async throws -> Data {
         logger.debug("Fetching documentation page", metadata: ["path": .string(path)])
         var url = baseURL
         for component in path.split(separator: "/") {
             url.append(component: component)
         }
         url.appendPathExtension("json")
-        let data = try await fetchData(from: url)
+        return try await fetchData(from: url)
+    }
+
+    func fetchDocumentationPage(path: String) async throws -> TechnologyDocumentationPageDTO {
+        try decodeDiscoveryPage(try await fetchDocumentationData(path: path), path: path)
+    }
+
+    private func decodeDiscoveryPage(_ data: Data, path: String) throws -> TechnologyDocumentationPageDTO {
         do {
             let page = try JSONDecoder().decode(TechnologyDocumentationPageDTO.self, from: data)
             logger.trace(
                 "Decoded documentation page", metadata: ["references": .stringConvertible(page.references.count)])
             return page
         } catch {
-            logger.error("Failed to decode documentation page", metadata: ["path": .string(url.path)])
+            logger.error("Failed to decode documentation page", metadata: ["path": .string(path)])
             throw error
         }
     }
@@ -199,17 +212,16 @@ struct DefaultAppleDocumentationClient<Dependencies: DefaultAppleDocumentationCl
 extension DefaultAppleDocumentationClient {
     struct DocumentationRoot: Sendable {
         let page: TechnologyDocumentationPageDTO
+        let document: TypeDocumentationDocument
         let slug: String
         let name: String
         let url: String
     }
 
     func fetchDocumentationRoot(technology: String) async throws -> DocumentationRoot {
         do {
-            return DocumentationRoot(
-                page: try await fetchDocumentationPage(path: "/documentation/\(technology.lowercased())"),
-                slug: technology,
-                name: technology,
+            return try await loadRoot(
+                slug: technology, name: technology,
                 url: "https://developer.apple.com/documentation/\(technology.lowercased())"
             )
         } catch Error.httpStatus(404) {
@@ -226,19 +238,25 @@ extension DefaultAppleDocumentationClient {
             }
             logger.debug("Retrying documentation root with canonical technology", metadata: ["slug": .string(slug)])
             do {
-                return DocumentationRoot(
-                    page: try await fetchDocumentationPage(path: "/documentation/\(slug.lowercased())"),
-                    slug: slug,
-                    name: resolved.name,
-                    url: resolved.url
-                )
+                return try await loadRoot(slug: slug, name: resolved.name, url: resolved.url)
             } catch Error.httpStatus(404) {
                 logger.notice("Canonical documentation root unavailable", metadata: ["slug": .string(slug)])
                 throw Error.unsupportedTechnology(name: resolved.name, url: resolved.url)
             }
         }
     }
 
+    private func loadRoot(slug: String, name: String, url: String) async throws -> DocumentationRoot {
+        let destination = DocumentationDestination(
+            technology: slug.lowercased(), path: "/documentation/\(slug.lowercased())"
+        )
+        let document = try await fetchDocument(at: destination)
+        return DocumentationRoot(
+            page: try decodeDiscoveryPage(document.data, path: destination.path), document: document,
+            slug: slug, name: name, url: url
+        )
+    }
+
     private func fetchData(from url: URL) async throws -> Data {
         let started = ContinuousClock.now
         logger.debug("Requesting documentation data", metadata: ["path": .string(url.path)])
@@ -282,16 +300,11 @@ extension DefaultAppleDocumentationClient {
         return data
     }
 
-    private func typeURL(name: String, technology: String) -> URL {
-        var url = baseURL.appending(component: "documentation")
-            .appending(component: technology.lowercased())
-        // DocC uses path components for nested symbols while Swift spelling uses dots.
-        for component in name.replacingOccurrences(of: ".", with: "/").split(separator: "/") {
-            url.append(component: component.lowercased())
-        }
-        url.appendPathExtension("json")
-        logger.trace("Resolved type documentation path", metadata: ["path": .string(url.path)])
-        return url
+    private func typeDestination(name: String, technology: String) -> DocumentationDestination {
+        // Only CLI Swift names use dots as hierarchy separators. Resolved links retain their exact paths.
+        let components = name.replacingOccurrences(of: ".", with: "/").split(separator: "/")
+        let path = "/documentation/\(technology.lowercased())/" + components.joined(separator: "/").lowercased()
+        return DocumentationDestination(technology: technology.lowercased(), path: path)
     }
 
     func resolveTechnology(named requestedName: String) async throws -> ResolvedTechnology {

Sources/CLI/client/TypeDocumentationDocument.swift

@@ -2,4 +2,5 @@ import Foundation
 
 struct TypeDocumentationDocument: Sendable {
     let data: Data
+    let destination: DocumentationDestination
 }

Sources/CLI/documentation/DocumentationDestination.swift

@@ -0,0 +1,79 @@
+import Foundation
+
+struct DocumentationDestination: Hashable, Sendable {
+    let technology: String
+    let path: String
+    var fragment: String?
+
+    init(technology: String, path: String, fragment: String? = nil) {
+        self.technology = technology
+        self.path = path
+        self.fragment = fragment
+    }
+
+    var url: URL {
+        var components = URLComponents()
+        components.scheme = "https"
+        components.host = "developer.apple.com"
+        components.path = path
+        components.fragment = fragment
+        // Internal destinations have already passed boundary validation.
+        return components.url!
+    }
+
+    static func resolve(
+        _ raw: String, relativeTo current: DocumentationDestination
+    ) throws -> DocumentationLinkTarget {
+        guard !raw.isEmpty, !raw.hasPrefix("//") else { throw DestinationError.invalidURL }
+        try validate(raw)
+        guard let input = URLComponents(string: raw), input.user == nil, input.password == nil else {
+            throw DestinationError.invalidURL
+        }
+        if input.scheme?.lowercased() == "doc" { return .unavailable(raw) }
+        if let scheme = input.scheme, !["https", "http"].contains(scheme.lowercased()) {
+            throw DestinationError.invalidURL
+        }
+        guard let url = URL(string: raw, relativeTo: current.url)?.absoluteURL,
+            let components = URLComponents(url: url, resolvingAgainstBaseURL: true),
+            let host = components.host, !host.isEmpty
+        else { throw DestinationError.invalidURL }
+
+        guard components.scheme?.lowercased() == "https", host.lowercased() == "developer.apple.com" else {
+            return .external(url)
+        }
+        guard components.port == nil || components.port == 443 else { throw DestinationError.invalidURL }
+        let parts = components.path.split(separator: "/", omittingEmptySubsequences: false)
+        guard parts.count >= 3, parts[1] == "documentation", !parts[2].isEmpty else {
+            return .external(url)
+        }
+        guard components.query == nil else { throw DestinationError.invalidURL }
+        return .documentation(
+            DocumentationDestination(
+                technology: parts[2].lowercased(), path: components.path, fragment: components.fragment
+            ))
+    }
+
+    private static func validate(_ raw: String) throws {
+        var value = raw
+        // Check each decoding layer before URL resolution can erase traversal components.
+        while true {
+            guard !value.contains("\\"), value.rangeOfCharacter(from: .controlCharacters) == nil,
+                let components = URLComponents(string: value),
+                !components.path.split(separator: "/").contains(where: { $0 == "." || $0 == ".." })
+            else { throw DestinationError.invalidURL }
+            guard let decoded = value.removingPercentEncoding else { throw DestinationError.invalidURL }
+            if decoded == value { return }
+            value = decoded
+        }
+    }
+
+    enum DestinationError: Error {
+        case invalidURL
+    }
+}
+
+enum DocumentationLinkTarget: Equatable, Sendable {
+    case documentation(DocumentationDestination)
+    case external(URL)
+    case unavailable(String)
+}

Sources/CLI/documentation/DocumentationPage.swift

@@ -0,0 +1,63 @@
+import Foundation
+
+struct DocumentationPage: Equatable, Sendable {
+    let destination: DocumentationDestination
+    let title: String
+    let kind: String
+    var symbolKind: String?
+    var roleHeading: String?
+    var modules: [String] = []
+    var abstract: [DocumentationInline] = []
+    var deprecation: [DocumentationBlock] = []
+    var declarations: [DocumentationDeclaration] = []
+    var availability: [DocumentationAvailability] = []
+    var content: [DocumentationBlock] = []
+    var relationships: [DocumentationGroup] = []
+    var topics: [DocumentationGroup] = []
+    var seeAlso: [DocumentationGroup] = []
+
+    var url: URL { destination.url }
+}
+
+indirect enum DocumentationInline: Equatable, Sendable {
+    case text(String)
+    case code(String)
+    case link(label: [DocumentationInline], target: DocumentationLinkTarget)
+}
+
+indirect enum DocumentationBlock: Equatable, Sendable {
+    case paragraph([DocumentationInline])
+    case heading(String)
+    case codeListing(code: [String], syntax: String?)
+    case orderedList(items: [[DocumentationBlock]], startIndex: Int)
+    case unorderedList([[DocumentationBlock]])
+    case aside(content: [DocumentationBlock], style: String, name: String?)
+}
+
+struct DocumentationDeclaration: Equatable, Sendable {
+    let languages: [String]
+    let text: String
+}
+
+struct DocumentationAvailability: Equatable, Sendable {
+    let name: String
+    let introducedAt: String?
+    let deprecatedAt: String?
+    var obsoletedAt: String?
+    var isBeta = false
+    var isUnavailable = false
+}
+
+struct DocumentationReference: Equatable, Sendable {
+    let id: String
+    let title: String
+    let kind: String
+    var abstract: [DocumentationInline] = []
+    let target: DocumentationLinkTarget
+}
+
+struct DocumentationGroup: Equatable, Sendable {
+    let id: String
+    let title: String
+    let references: [DocumentationReference]
+}