diff --git a/LoopLibCore/sources/pdffont.cpp b/LoopLibCore/sources/pdffont.cpp index e1aff6460..282d4628f 100644 --- a/LoopLibCore/sources/pdffont.cpp +++ b/LoopLibCore/sources/pdffont.cpp @@ -1217,10 +1217,12 @@ void PDFRealizedFontImpl::fillTextSequence(const QByteArray& byteArray, TextSequ { const Glyph& glyph = getGlyph(glyphIndex); textSequence.items.emplace_back(&glyph.glyph, font->getUnicode(cid), glyph.advance, cid); + textSequence.items.back().glyphIndex = glyphIndex; } else { reporter->reportRenderError(RenderErrorType::Warning, PDFTranslationContext::tr("Glyph for simple font character code '%1' not found.").arg(cid)); + textSequence.unresolvedCodes.push_back(cid); if (glyphWidth > 0) { const QPainterPath* nullpath = nullptr; @@ -1275,6 +1277,7 @@ void PDFRealizedFontImpl::fillTextSequence(const QByteArray& byteArray, TextSequ { const Glyph& glyph = getGlyph(*glyphIndex); textSequence.items.emplace_back(&glyph.glyph, character, glyph.advance, cid); + textSequence.items.back().glyphIndex = *glyphIndex; } else { @@ -1282,6 +1285,7 @@ void PDFRealizedFontImpl::fillTextSequence(const QByteArray& byteArray, TextSequ { // Character with CID == 0 is treated as default whitespace, it hasn't glyph reporter->reportRenderError(RenderErrorType::Warning, PDFTranslationContext::tr("Glyph for composite font character with cid '%1' not found.").arg(cid)); + textSequence.unresolvedCodes.push_back(cid); } if (glyphWidth > 0) diff --git a/LoopLibCore/sources/pdffont.h b/LoopLibCore/sources/pdffont.h index 4d6a56abe..bf1f25acb 100644 --- a/LoopLibCore/sources/pdffont.h +++ b/LoopLibCore/sources/pdffont.h @@ -79,9 +79,25 @@ class ITreeFactory struct TextSequenceItem { inline explicit TextSequenceItem() = default; - inline explicit TextSequenceItem(const QPainterPath* glyph, QChar character, PDFReal advance, CID cid) : glyph(glyph), character(character), advance(advance), cid(cid) { } - inline explicit TextSequenceItem(PDFReal advance) : character(), advance(advance) { } - inline explicit TextSequenceItem(const QByteArray* characterContentStream, QChar character, PDFReal advance, uint cid) : characterContentStream(characterContentStream), character(character), advance(advance), cid(cid) { } + inline explicit TextSequenceItem(const QPainterPath* glyph, QChar character, PDFReal advance, CID cid) : + glyph(glyph), + character(character), + advance(advance), + cid(cid) + { + } + inline explicit TextSequenceItem(PDFReal advance) : + character(), + advance(advance) + { + } + inline explicit TextSequenceItem(const QByteArray* characterContentStream, QChar character, PDFReal advance, uint cid) : + characterContentStream(characterContentStream), + character(character), + advance(advance), + cid(cid) + { + } inline bool isContentStream() const { return characterContentStream; } inline bool isCharacter() const { return glyph; } @@ -93,11 +109,18 @@ struct TextSequenceItem QChar character; PDFReal advance = 0; CID cid = 0; + /// Glyph index in the font program; 0 is .notdef. Zero also for advances and Type 3 glyphs. + GID glyphIndex = 0; }; struct TextSequence { std::vector items; + + /// Shown codes (character codes of simple fonts, CIDs of composite fonts) that + /// resolved to no glyph, whatever their advance. Composite CID 0 is the default + /// whitespace and is never listed. + std::vector unresolvedCodes; }; constexpr bool isTextRenderingModeFilled(TextRenderingMode mode) @@ -295,7 +318,10 @@ class LOOPLIBCORESHARED_EXPORT PDFRealizedFont private: /// Constructs new realized font - explicit PDFRealizedFont(IRealizedFontImpl* impl) : m_impl(impl) { } + explicit PDFRealizedFont(IRealizedFontImpl* impl) : + m_impl(impl) + { + } IRealizedFontImpl* m_impl; }; @@ -417,7 +443,7 @@ class LOOPLIBCORESHARED_EXPORT PDFSimpleFont : public PDFFont bool m_hasToUnicode; GlyphIndices m_glyphIndices; GlyphNames m_glyphNames; - StandardFontType m_standardFontType; ///< Type of the standard font (or invalid, if it is not a standard font) + StandardFontType m_standardFontType; ///< Type of the standard font (or invalid, if it is not a standard font) }; class PDFType1Font : public PDFSimpleFont @@ -468,7 +494,6 @@ class LOOPLIBCORESHARED_EXPORT PDFFontCache m_realizedFontCacheLimit(realizedFontCacheLimit), m_document(nullptr) { - } ~PDFFontCache(); @@ -543,7 +568,10 @@ class LOOPLIBCORESHARED_EXPORT PDFFontCache class PDFCIDtoGIDMapper { public: - explicit inline PDFCIDtoGIDMapper(QByteArray&& mapping) : m_mapping(qMove(mapping)) { } + explicit inline PDFCIDtoGIDMapper(QByteArray&& mapping) : + m_mapping(qMove(mapping)) + { + } /// Maps CID to GID (glyph identifier). Nullopt means no valid mapping exists. std::optional tryMap(CID cid) const @@ -669,11 +697,16 @@ class LOOPLIBCORESHARED_EXPORT PDFFontCMap bool containsCode(unsigned int code, unsigned int byteCount) const; private: - struct Entry { constexpr explicit inline Entry() = default; - constexpr explicit inline Entry(unsigned int from, unsigned int to, unsigned int byteCount, CID cid) : from(from), to(to), byteCount(byteCount), cid(cid) { } + constexpr explicit inline Entry(unsigned int from, unsigned int to, unsigned int byteCount, CID cid) : + from(from), + to(to), + byteCount(byteCount), + cid(cid) + { + } unsigned int from = 0; unsigned int to = 0; @@ -777,7 +810,6 @@ class LOOPLIBCORESHARED_EXPORT PDFType0Font : public PDFFont m_defaultAdvance(defaultAdvance), m_advances(qMove(advances)) { - } virtual ~PDFType0Font() = default; @@ -845,4 +877,4 @@ class LOOPLIBCORESHARED_EXPORT PDFSystemFont } // namespace pdf -#endif // PDFFONT_H +#endif // PDFFONT_H diff --git a/LoopLibCore/sources/pdffontintegrity.cpp b/LoopLibCore/sources/pdffontintegrity.cpp index 45e3570f7..dd4b36ee3 100644 --- a/LoopLibCore/sources/pdffontintegrity.cpp +++ b/LoopLibCore/sources/pdffontintegrity.cpp @@ -24,6 +24,7 @@ #include "pdffont.h" +#include #include #include @@ -41,10 +42,7 @@ quint16 readU16(const QByteArray& data, int offset) quint32 readU32(const QByteArray& data, int offset) { - return (quint32(uchar(data.at(offset))) << 24) - | (quint32(uchar(data.at(offset + 1))) << 16) - | (quint32(uchar(data.at(offset + 2))) << 8) - | quint32(uchar(data.at(offset + 3))); + return (quint32(uchar(data.at(offset))) << 24) | (quint32(uchar(data.at(offset + 1))) << 16) | (quint32(uchar(data.at(offset + 2))) << 8) | quint32(uchar(data.at(offset + 3))); } void inspectTrueType(const QByteArray& program, FontType fontType, QStringList& defects) @@ -56,8 +54,7 @@ void inspectTrueType(const QByteArray& program, FontType fontType, QStringList& } const QByteArray magic = program.left(4); - if (magic != QByteArrayLiteral("OTTO") - && magic != QByteArray::fromHex("00010000")) + if (magic != QByteArrayLiteral("OTTO") && magic != QByteArray::fromHex("00010000")) { defects.append(QStringLiteral("UnreadableTableDirectory")); return; @@ -79,8 +76,7 @@ void inspectTrueType(const QByteArray& program, FontType fontType, QStringList& tables.insert(tag); const quint32 length = readU32(program, offset + 12); const quint32 tableOffset = readU32(program, offset + 8); - if (tableOffset > quint32(program.size()) - || length > quint32(program.size()) - tableOffset) + if (tableOffset > quint32(program.size()) || length > quint32(program.size()) - tableOffset) { defects.append(QStringLiteral("TruncatedProgram")); continue; @@ -96,14 +92,13 @@ void inspectTrueType(const QByteArray& program, FontType fontType, QStringList& } } - if (fontType == FontType::TrueType && tables.contains(QByteArrayLiteral("glyf")) - && !tables.contains(QByteArrayLiteral("loca"))) + if (fontType == FontType::TrueType && tables.contains(QByteArrayLiteral("glyf")) && !tables.contains(QByteArrayLiteral("loca"))) { defects.append(QStringLiteral("GlyfLocaInconsistent")); } } -} // namespace +} // namespace PDFFontIntegrityResult inspectPDFFontIntegrity(const PDFFont& font) { @@ -130,8 +125,7 @@ PDFFontIntegrityResult inspectPDFFontIntegrity(const PDFFont& font) break; case FontType::Type1: case FontType::MMType1: - if (!program->startsWith("%!") - && !(program->size() >= 2 && uchar(program->at(0)) == 0x80 && uchar(program->at(1)) == 0x01)) + if (!program->startsWith("%!") && !(program->size() >= 2 && uchar(program->at(0)) == 0x80 && uchar(program->at(1)) == 0x01)) { result.defects.append(QStringLiteral("UnreadableType1Program")); } @@ -151,4 +145,47 @@ PDFFontIntegrityResult inspectPDFFontIntegrity(const PDFFont& font) return result; } -} // namespace pdf +PDFShownGlyphDefect classifyShownGlyph(const TextSequenceItem& item) +{ + if (item.isContentStream()) + { + return PDFShownGlyphDefect::None; + } + + if (!item.glyph) + { + // Codes that resolve to nothing are listed in TextSequence::unresolvedCodes. + return PDFShownGlyphDefect::None; + } + + if (item.glyphIndex == 0) + { + return PDFShownGlyphDefect::Notdef; + } + + const bool visibleCharacter = !item.character.isNull() && !item.character.isSpace() && item.character.category() != QChar::Other_Format && item.character.category() != QChar::Other_Control; + if (visibleCharacter && item.glyph->isEmpty()) + { + return PDFShownGlyphDefect::EmptyOutline; + } + + return PDFShownGlyphDefect::None; +} + +QString shownGlyphDefectName(PDFShownGlyphDefect defect) +{ + switch (defect) + { + case PDFShownGlyphDefect::Unresolved: + return QStringLiteral("MissingGlyph"); + case PDFShownGlyphDefect::Notdef: + return QStringLiteral("NotdefGlyph"); + case PDFShownGlyphDefect::EmptyOutline: + return QStringLiteral("EmptyGlyph"); + case PDFShownGlyphDefect::None: + break; + } + return QString(); +} + +} // namespace pdf diff --git a/LoopLibCore/sources/pdffontintegrity.h b/LoopLibCore/sources/pdffontintegrity.h index 63ed67116..10f48e555 100644 --- a/LoopLibCore/sources/pdffontintegrity.h +++ b/LoopLibCore/sources/pdffontintegrity.h @@ -31,6 +31,7 @@ namespace pdf { class PDFFont; +struct TextSequenceItem; struct LOOPLIBCORESHARED_EXPORT PDFFontIntegrityResult { @@ -46,6 +47,21 @@ struct LOOPLIBCORESHARED_EXPORT PDFFontIntegrityResult /// does not alter the existing embedded-fonts check contract. LOOPLIBCORESHARED_EXPORT PDFFontIntegrityResult inspectPDFFontIntegrity(const PDFFont& font); -} // namespace pdf +enum class PDFShownGlyphDefect +{ + None, + Unresolved, ///< The shown code resolved to no glyph (TextSequence::unresolvedCodes). + Notdef, ///< The shown code resolved to glyph 0 (.notdef). + EmptyOutline ///< The glyph exists but draws nothing for a visible character. +}; + +/// Classifies one resolved glyph item of a text sequence. Advances (TJ +/// adjustments), glyphless items and Type 3 glyph procedures are never defects +/// here; codes that resolved to nothing are reported by the text sequence. +LOOPLIBCORESHARED_EXPORT PDFShownGlyphDefect classifyShownGlyph(const TextSequenceItem& item); + +LOOPLIBCORESHARED_EXPORT QString shownGlyphDefectName(PDFShownGlyphDefect defect); + +} // namespace pdf -#endif // PDFFONTINTEGRITY_H +#endif // PDFFONTINTEGRITY_H diff --git a/LoopLibCore/sources/pdfpagecontentprocessor.cpp b/LoopLibCore/sources/pdfpagecontentprocessor.cpp index e8c55e476..1cda52c4c 100644 --- a/LoopLibCore/sources/pdfpagecontentprocessor.cpp +++ b/LoopLibCore/sources/pdfpagecontentprocessor.cpp @@ -540,6 +540,11 @@ void PDFPageContentProcessor::performProcessTextSequence(const TextSequence& tex Q_UNUSED(order); } +void PDFPageContentProcessor::performTextGlyphsUnresolved(const TextSequence& textSequence) +{ + Q_UNUSED(textSequence); +} + bool PDFPageContentProcessor::isContentKindSuppressed(ContentKind kind) const { Q_UNUSED(kind); @@ -3406,6 +3411,11 @@ void PDFPageContentProcessor::operatorCompatibilityEnd() void PDFPageContentProcessor::drawText(const TextSequence& textSequence) { + if (!textSequence.unresolvedCodes.empty()) + { + performTextGlyphsUnresolved(textSequence); + } + if (textSequence.items.empty()) { // Do not display empty text diff --git a/LoopLibCore/sources/pdfpagecontentprocessor.h b/LoopLibCore/sources/pdfpagecontentprocessor.h index af1ab4703..700a86429 100644 --- a/LoopLibCore/sources/pdfpagecontentprocessor.h +++ b/LoopLibCore/sources/pdfpagecontentprocessor.h @@ -753,6 +753,10 @@ class LOOPLIBCORESHARED_EXPORT PDFPageContentProcessor : public PDFRenderErrorRe /// Implement to respond to text sequence processing virtual void performProcessTextSequence(const TextSequence& textSequence, ProcessOrder order); + /// Called once per shown string for which at least one code resolved to no + /// glyph, including strings that produce no drawable items at all. + virtual void performTextGlyphsUnresolved(const TextSequence& textSequence); + enum class ContentKind { Shapes, ///< General shapes (they can be also shaded / tiled) diff --git a/LoopLibCore/sources/preflightengine.cpp b/LoopLibCore/sources/preflightengine.cpp index d0d7dcc24..b2b01945f 100644 --- a/LoopLibCore/sources/preflightengine.cpp +++ b/LoopLibCore/sources/preflightengine.cpp @@ -74,6 +74,7 @@ #include #include #include +#include #include namespace pdf @@ -5052,6 +5053,118 @@ void runEmbeddedFontsCheck(PDFDocumentSession* session, } } +struct ShownGlyphDefects +{ + PDFFontPointer font; + QString fontName; + QString subtype; + bool composite = false; + std::map> codesByDefect; +}; + +/// Records, per embedded font, the codes a page actually shows that do not +/// resolve to a usable glyph. Page content, Form XObjects and annotation +/// appearance streams all flow through the same text-sequence hook. +class ShownGlyphCoverageProcessor final : public PDFPageContentProcessor +{ +public: + using PDFPageContentProcessor::PDFPageContentProcessor; + + void processFormStream(const PDFStream* stream) + { + if (stream && !isContentSuppressed()) + { + processForm(stream); + } + } + + const std::vector& defects() const { return m_defects; } + +protected: + bool isContentKindSuppressed(ContentKind kind) const override + { + return kind != ContentKind::Text && kind != ContentKind::Forms; + } + + void performTextGlyphsUnresolved(const TextSequence& textSequence) override + { + const PDFFontPointer font = embeddedShownFont(); + if (!font) + { + return; + } + + for (const CID code : textSequence.unresolvedCodes) + { + record(font, PDFShownGlyphDefect::Unresolved, code); + } + } + + void performProcessTextSequence(const TextSequence& textSequence, ProcessOrder order) override + { + if (order != ProcessOrder::BeforeOperation) + { + return; + } + + const PDFFontPointer font = embeddedShownFont(); + if (!font) + { + return; + } + + for (const TextSequenceItem& item : textSequence.items) + { + const PDFShownGlyphDefect defect = classifyShownGlyph(item); + if (defect == PDFShownGlyphDefect::None) + { + continue; + } + + record(font, defect, item.cid); + } + } + +private: + /// Invisible (Tr 3) and clip-only (Tr 7) text shows nothing, so OCR text layers + /// drawn with glyphless fonts are not audited. + PDFFontPointer embeddedShownFont() const + { + const TextRenderingMode mode = getGraphicState()->getTextRenderingMode(); + if (!isTextRenderingModeFilled(mode) && !isTextRenderingModeStroked(mode)) + { + return nullptr; + } + + const PDFFontPointer font = getGraphicState()->getTextFont(); + if (!font || !font->getFontDescriptor() || !font->getFontDescriptor()->isEmbedded() || font->getFontType() == FontType::Type3) + { + return nullptr; + } + return font; + } + + /// Entries are per font object, not per resource name: page and form resources + /// may bind the same name to different fonts. + void record(const PDFFontPointer& font, PDFShownGlyphDefect defect, CID code) + { + auto entry = std::find_if(m_defects.begin(), m_defects.end(), [&font](const ShownGlyphDefects& defects) + { return defects.font == font; }); + if (entry == m_defects.end()) + { + ShownGlyphDefects defects; + defects.font = font; + defects.fontName = QString::fromLatin1(font->getFontId()); + defects.subtype = QString::number(static_cast(font->getFontType())); + defects.composite = font->getFontType() == FontType::Type0; + entry = m_defects.insert(m_defects.end(), std::move(defects)); + } + entry->codesByDefect[shownGlyphDefectName(defect)].insert(code); + } + + std::vector m_defects; +}; + // LOW CONFIDENCE NOTE: DPI calculation uses getCurrentTransformationMatrix() // from the PDFPageContentProcessor state, which is in PDF user space. // This matches the existing PDFImageCollectorProcessor pattern in @@ -5188,6 +5301,108 @@ void runFontIntegrityCheck(PDFDocumentSession* session, scanResources(page->getResources(), int(pageIndex + 1)); } } + + // Shown-glyph coverage: every code the pages, forms and annotation appearances + // actually show must resolve to a real glyph in the embedded program. + PDFOptionalContentActivity ocActivity(document, OCUsage::Export, nullptr); + PDFFontCache fontCache(DEFAULT_FONT_CACHE_LIMIT, DEFAULT_REALIZED_FONT_CACHE_LIMIT); + PDFModifiedDocument modifiedDocument(document, &ocActivity); + fontCache.setDocument(modifiedDocument); + fontCache.setCacheShrinkEnabled(nullptr, false); + PDFCMSManager cmsManager(nullptr); + cmsManager.setDocument(document); + PDFCMSPointer cms = cmsManager.getCurrentCMS(); + PDFMeshQualitySettings meshQuality; + + for (PDFInteger pageIndex = 0; pageIndex < pageCount; ++pageIndex) + { + const PDFPage* page = document->getCatalog()->getPage(pageIndex); + if (!page) + { + continue; + } + + const int pageNumber = int(pageIndex + 1); + std::vector pageDefects; + bool incomplete = false; + QString incompleteReason; + try + { + ShownGlyphCoverageProcessor processor(page, document, &fontCache, cms.get(), &ocActivity, + QTransform(), meshQuality, session->getProcessingBudget()); + processor.processContents(); + processAnnotationAppearanceStreams(document, page, pageNumber, [&](const PDFPage*, const PDFStream* formStream) + { processor.processFormStream(formStream); }); + pageDefects = processor.defects(); + std::stable_sort(pageDefects.begin(), pageDefects.end(), [](const ShownGlyphDefects& left, const ShownGlyphDefects& right) + { return left.fontName < right.fontName; }); + } + catch (const PDFBudgetExceededException&) + { + throw; + } + catch (const PDFException& exception) + { + incomplete = true; + incompleteReason = QString::fromUtf8(exception.what()); + } + + for (const ShownGlyphDefects& shown : pageDefects) + { + const QString& fontName = shown.fontName; + QStringList defectNames; + QStringList codeText; + QJsonArray missingCodes; + std::set allCodes; + for (const auto& [defectName, codes] : shown.codesByDefect) + { + defectNames.append(defectName); + allCodes.insert(codes.begin(), codes.end()); + } + for (unsigned int code : allCodes) + { + missingCodes.append(int(code)); + codeText.append(QString::number(code)); + } + + PreflightFinding finding; + finding.scope = QString::fromLatin1(PREFLIGHT_FINDING_SCOPE_PAGE); + finding.page = pageNumber; + finding.type = QStringLiteral("font-integrity"); + finding.checkId = check.id; + finding.severity = check.severity; + finding.message = PDFTranslationContext::tr("Font '%1' on page %2 has no usable glyph for shown %3: %4") + .arg(fontName) + .arg(pageNumber) + .arg(shown.composite ? QStringLiteral("CIDs") : QStringLiteral("character codes"), + codeText.join(QStringLiteral(", "))); + finding.evidence.insert(QStringLiteral("font_resource"), fontName); + finding.evidence.insert(QStringLiteral("font_subtype"), shown.subtype); + finding.evidence.insert(QStringLiteral("embedded"), true); + finding.evidence.insert(QStringLiteral("inspection_complete"), true); + finding.evidence.insert(QStringLiteral("code_kind"), + shown.composite ? QStringLiteral("cid") : QStringLiteral("character-code")); + finding.evidence.insert(QStringLiteral("missing_codes"), missingCodes); + finding.evidence.insert(QStringLiteral("defects"), QJsonArray::fromStringList(defectNames)); + pushPreflightFinding(finding, finding.severity, errors, warnings); + } + + if (incomplete) + { + PreflightFinding finding; + finding.scope = QString::fromLatin1(PREFLIGHT_FINDING_SCOPE_PAGE); + finding.page = pageNumber; + finding.type = QStringLiteral("font-integrity"); + finding.checkId = check.id; + finding.severity = QStringLiteral("error"); + finding.message = PDFTranslationContext::tr("Shown-glyph coverage could not be audited on page %1: %2") + .arg(pageNumber) + .arg(incompleteReason); + finding.evidence.insert(QStringLiteral("inspection_complete"), false); + finding.evidence.insert(QStringLiteral("defects"), QJsonArray{ QStringLiteral("GlyphCoverageIncomplete") }); + pushPreflightFinding(finding, finding.severity, errors, warnings); + } + } } PDFXRuleResult makePDFXRuleResult(const QString& ruleId, diff --git a/UnitTests/tst_preflightenginetest.cpp b/UnitTests/tst_preflightenginetest.cpp index a9fcd0294..94582cbee 100644 --- a/UnitTests/tst_preflightenginetest.cpp +++ b/UnitTests/tst_preflightenginetest.cpp @@ -32,6 +32,8 @@ #include "pdfimage.h" #include "pdfinkcoverageprobe.h" #include "pdffixupregistry.h" +#include "pdffont.h" +#include "pdffontintegrity.h" #include "pdfrepairoperation.h" #include "pdfobject.h" #include "pdfthinpartprobe.h" @@ -82,6 +84,8 @@ private slots: void thinPartProbe_reportsBoundedWidthAndPrecision(); void fontIntegrity_checkIsRegistered(); void run_fontIntegrity_keepsValidEmbeddedFixtureClean(); + void run_fontIntegrity_reportsShownGlyphsMissingFromSubset(); + void classifyShownGlyph_separatesDefectsFromAdvancesAndSpaces(); void hiddenContent_checksAreRegistered(); void run_offPageContent_detectsMarksOutsideToleratedBox(); void run_includesProfileFixups(); @@ -901,6 +905,63 @@ void PreflightEngineTest::run_fontIntegrity_keepsValidEmbeddedFixtureClean() QVERIFY(result.warnings.isEmpty()); } +void PreflightEngineTest::run_fontIntegrity_reportsShownGlyphsMissingFromSubset() +{ + // The subset parses cleanly but has no glyph for codes 0xE9 (page content), + // 0xEA (Form XObject) and 0xEB (annotation appearance stream). + const QString fixturePath = QStringLiteral(LOOP_PREFLIGHT_SOURCE_DIR "/testdata/fixtures/font-glyph-missing.pdf"); + QVERIFY(QFile::exists(fixturePath)); + + pdf::PDFDocumentReader reader(nullptr, [](bool*) + { return QString(); }, true, false); + pdf::PDFDocument document = reader.readFromFile(fixturePath); + QCOMPARE(reader.getReadingResult(), pdf::PDFDocumentReader::Result::OK); + + pdf::PDFDocumentSession session(&document); + pdf::PreflightEngine engine(&session); + const QJsonObject profile{ + { QStringLiteral("name"), QStringLiteral("Font integrity") }, + { QStringLiteral("checks"), QJsonArray{ + QJsonObject{ { QStringLiteral("id"), QStringLiteral("font-integrity") } } } } + }; + + const pdf::PreflightResult result = engine.run(profile); + QVERIFY(!result.pass); + QCOMPARE(result.errors.size(), 1); + + const pdf::PreflightFinding& finding = result.errors.first(); + QCOMPARE(finding.checkId, QStringLiteral("font-integrity")); + QCOMPARE(finding.page, 1); + QCOMPARE(finding.evidence.value(QStringLiteral("font_resource")).toString(), QStringLiteral("F2+0")); + QCOMPARE(finding.evidence.value(QStringLiteral("inspection_complete")).toBool(), true); + QCOMPARE(finding.evidence.value(QStringLiteral("code_kind")).toString(), QStringLiteral("character-code")); + QCOMPARE(finding.evidence.value(QStringLiteral("missing_codes")).toArray(), (QJsonArray{ 0xE9, 0xEA, 0xEB })); + QVERIFY(finding.evidence.value(QStringLiteral("defects")).toArray().contains(QStringLiteral("MissingGlyph"))); +} + +void PreflightEngineTest::classifyShownGlyph_separatesDefectsFromAdvancesAndSpaces() +{ + const QPainterPath* noGlyph = nullptr; + const QPainterPath empty; + QPainterPath outline; + outline.addRect(0, 0, 1, 1); + + const auto glyphItem = [](const QPainterPath* path, QChar character, pdf::GID gid) + { + pdf::TextSequenceItem item(path, character, 500.0, 7); + item.glyphIndex = gid; + return item; + }; + + QVERIFY(pdf::classifyShownGlyph(pdf::TextSequenceItem(-120.0)) == pdf::PDFShownGlyphDefect::None); + QVERIFY(pdf::classifyShownGlyph(pdf::TextSequenceItem(noGlyph, QChar(), 500.0, 7)) == pdf::PDFShownGlyphDefect::None); + QVERIFY(pdf::classifyShownGlyph(glyphItem(&outline, QChar('A'), 0)) == pdf::PDFShownGlyphDefect::Notdef); + QVERIFY(pdf::classifyShownGlyph(glyphItem(&empty, QChar('A'), 12)) == pdf::PDFShownGlyphDefect::EmptyOutline); + QVERIFY(pdf::classifyShownGlyph(glyphItem(&empty, QChar(' '), 12)) == pdf::PDFShownGlyphDefect::None); + QVERIFY(pdf::classifyShownGlyph(glyphItem(&empty, QChar(), 12)) == pdf::PDFShownGlyphDefect::None); + QVERIFY(pdf::classifyShownGlyph(glyphItem(&outline, QChar('A'), 12)) == pdf::PDFShownGlyphDefect::None); +} + void PreflightEngineTest::hiddenContent_checksAreRegistered() { pdf::PreflightEngine engine(nullptr); diff --git a/changes/cc-issue-114-glyph-coverage.evidence.yaml b/changes/cc-issue-114-glyph-coverage.evidence.yaml new file mode 100644 index 000000000..378dffa97 --- /dev/null +++ b/changes/cc-issue-114-glyph-coverage.evidence.yaml @@ -0,0 +1,29 @@ +format_version: 1 +kind: evidence +claims: + - id: font-glyph-coverage-regression-fixture + evidence: + - integration:UnitTestsPreflightCorpus + - architecture:loop-preflight/testdata/fixtures + - architecture:docs/generated/preflight-check-catalog.json + - architecture:docs/generated/architecture-catalog.json + - architecture:docs/generated/preflight-corpus-coverage.json + - id: shown-code-glyph-audit + evidence: + - unit:agent-policy:core + - unit:agent-policy:preflight + - unit:UnitTestsPreflightChecks + - unit:UnitTestsPreflightEngine + - unit:UnitTestsPreflightInteraction + - unit:UnitTestsPreflightVerdict + - unit:UnitTestsPreflightProfileResolver + - unit:UnitTestsProfileIdentity + - unit:UnitTestsOperatorAcceptance + - integration:UnitTestsPreflightWorkflowAcceptance + - differential:UnitTestsStandardOracle + - differential:UnitTestsConversionOracle + - security:scripts/ci/check_source_integrity.py +unresolved: + - core:scripts/ci/check_independent_validation_gate.py + - Only UnitTestsPreflightEngine and UnitTestsPreflightCorpus were built and run locally; the other listed unit, integration and differential lanes are proven by the linux-build and windows-build CI lanes, not by this worktree. + - Composite CID 0 (default whitespace) and Type 3 glyph procedures are not audited; Type 1 and CFF glyph-name coverage relies on the realized font's name lookup. diff --git a/changes/cc-issue-114-glyph-coverage.md b/changes/cc-issue-114-glyph-coverage.md new file mode 100644 index 000000000..fc8666720 --- /dev/null +++ b/changes/cc-issue-114-glyph-coverage.md @@ -0,0 +1,6 @@ +# Audit glyph coverage of shown codes in embedded fonts (#114) + +Category: fixed +Audience: prepress operators and preflight maintainers relying on font-integrity +Breaking-Change: no +Summary: font-integrity now resolves every character code or CID shown in page content, Form XObjects and annotation appearance streams and reports a finding, naming page, font and missing codes, when one maps to no glyph, .notdef, or an empty outline, so an embedded subset that parses but lacks a used glyph no longer passes clean. Adds the font-glyph-missing regression fixture and lands the font-glyph-coverage backlog row. diff --git a/docs/generated/preflight-check-catalog.json b/docs/generated/preflight-check-catalog.json index 700197bf2..66105fe4e 100644 --- a/docs/generated/preflight-check-catalog.json +++ b/docs/generated/preflight-check-catalog.json @@ -378,19 +378,21 @@ "evidence.font_subtype", "evidence.embedded", "evidence.inspection_complete", - "evidence.defects" + "evidence.defects", + "evidence.code_kind", + "evidence.missing_codes" ], "families": [ "sheetfed-offset", "digital" ], "fixups": [], - "limitations": "Not a full glyph-coverage audit.", - "measures": "Embedded font bytes that fail to parse or cmap.", + "limitations": "Glyph coverage audits codes shown in page content, Form XObjects and annotation appearance streams of embedded TrueType, Type 1 and CID fonts. Composite CID 0 (default whitespace) and Type 3 glyph procedures are not reported; a page whose content cannot be processed is reported incomplete.", + "measures": "Embedded font bytes that fail to parse or cmap, and shown character codes or CIDs that resolve to no glyph, .notdef, or an empty outline.", "parameters": [], "severity": [ { - "condition": "an embedded font program fails to parse or its cmap cannot be read; a complete inspection keeps the check severity, an incomplete one is forced to error", + "condition": "an embedded font program fails to parse or its cmap cannot be read, or a shown code resolves to no usable glyph; a complete inspection keeps the check severity, an incomplete one is forced to error", "finding_type": "font-integrity", "severity": "error" } diff --git a/docs/generated/preflight-corpus-coverage.json b/docs/generated/preflight-corpus-coverage.json index c261ad5f0..2d90965ee 100644 --- a/docs/generated/preflight-corpus-coverage.json +++ b/docs/generated/preflight-corpus-coverage.json @@ -120,6 +120,7 @@ "corpus_gap": null, "coverage": "partial", "finding_fixtures": [ + "font-glyph-missing", "font-integrity-corrupt" ], "uninspected_fixtures": [] @@ -272,6 +273,6 @@ } }, "snapshot_dir": "loop-preflight/testdata/snapshots", - "source_fixtures": 76, + "source_fixtures": 77, "unattributed_findings": 1 } diff --git a/docs/generated/preflight-coverage-backlog.json b/docs/generated/preflight-coverage-backlog.json index 8c428da56..92020d5fc 100644 --- a/docs/generated/preflight-coverage-backlog.json +++ b/docs/generated/preflight-coverage-backlog.json @@ -287,16 +287,16 @@ "state": "open" }, { - "closed_by": "#114", + "closed_by": "font-integrity", "deferral": null, "families": [ "sheetfed-offset", "digital" ], - "gap": "font-integrity parses the embedded program and its cmap but is not a glyph-coverage audit, so a missing glyph that still parses passes clean", + "gap": "shown character codes that resolve to no glyph, .notdef, or an empty outline in an embedded font that still parses were unchecked; font-integrity now audits them per page, form and annotation appearance (filed as #114)", "id": "font-glyph-coverage", "priority": "P2", - "state": "open" + "state": "landed" }, { "closed_by": "thin-strokes", diff --git a/docs/preflight-check-catalog-overlay.json b/docs/preflight-check-catalog-overlay.json index 916f07a9d..1b2f6e26e 100644 --- a/docs/preflight-check-catalog-overlay.json +++ b/docs/preflight-check-catalog-overlay.json @@ -517,8 +517,8 @@ "fixups": [] }, "font-integrity": { - "measures": "Embedded font bytes that fail to parse or cmap.", - "limitations": "Not a full glyph-coverage audit.", + "measures": "Embedded font bytes that fail to parse or cmap, and shown character codes or CIDs that resolve to no glyph, .notdef, or an empty outline.", + "limitations": "Glyph coverage audits codes shown in page content, Form XObjects and annotation appearance streams of embedded TrueType, Type 1 and CID fonts. Composite CID 0 (default whitespace) and Type 3 glyph procedures are not reported; a page whose content cannot be processed is reported incomplete.", "coverage": "partial", "families": [ "sheetfed-offset", @@ -529,7 +529,7 @@ { "finding_type": "font-integrity", "severity": "error", - "condition": "an embedded font program fails to parse or its cmap cannot be read; a complete inspection keeps the check severity, an incomplete one is forced to error" + "condition": "an embedded font program fails to parse or its cmap cannot be read, or a shown code resolves to no usable glyph; a complete inspection keeps the check severity, an incomplete one is forced to error" } ], "evidence": [ @@ -544,7 +544,9 @@ "evidence.font_subtype", "evidence.embedded", "evidence.inspection_complete", - "evidence.defects" + "evidence.defects", + "evidence.code_kind", + "evidence.missing_codes" ], "fixups": [] }, @@ -1482,13 +1484,13 @@ { "id": "font-glyph-coverage", "priority": "P2", - "gap": "font-integrity parses the embedded program and its cmap but is not a glyph-coverage audit, so a missing glyph that still parses passes clean", + "gap": "shown character codes that resolve to no glyph, .notdef, or an empty outline in an embedded font that still parses were unchecked; font-integrity now audits them per page, form and annotation appearance (filed as #114)", "families": [ "sheetfed-offset", "digital" ], - "state": "open", - "closed_by": "#114", + "state": "landed", + "closed_by": "font-integrity", "deferral": null }, { diff --git a/loop-preflight/README.md b/loop-preflight/README.md index 961836f20..0e94e8b8b 100644 --- a/loop-preflight/README.md +++ b/loop-preflight/README.md @@ -83,6 +83,14 @@ defects with the font resource and object reference, and marks unsupported formats as incomplete rather than clean. Existing `embedded-fonts` ids and severity behavior are unchanged. +It also audits glyph coverage: every character code or CID shown in page +content, Form XObjects and annotation appearance streams is resolved through +the font's encoding and cmap, and a code that resolves to no glyph, to +`.notdef`, or to an empty outline for a visible character is reported per page +and font with `evidence.missing_codes`. Composite CID 0 (default whitespace) and +Type 3 glyph procedures are not reported; a page whose content cannot +be processed is reported with `inspection_complete: false`. + ## Hidden and non-printing content The detection-only checks `invisible-content`, `hidden-layers`, @@ -581,6 +589,7 @@ Corpus-gap fixtures added for #668 (each isolates one previously unexercised che | `obscured-content.pdf` | test-obscured-content | info | `obscured-content` | | `hidden-layers.pdf` | test-hidden-layers | warning | `hidden-layers` | | `font-integrity-corrupt.pdf` | test-font-integrity | fail | `font-integrity` | +| `font-glyph-missing.pdf` | test-font-integrity | fail | `font-integrity` (shown codes absent from the subset) | | `thin-parts-clear.pdf` | test-thin-parts-clear | fail | `thin-parts` (not near-threshold) | | `blank-page.pdf` | test-dieline-required | fail | `dieline` | | `blank-page.pdf` | test-processing-steps-required | fail | `processing-steps` | diff --git a/loop-preflight/testdata/fixtures/font-glyph-missing.pdf b/loop-preflight/testdata/fixtures/font-glyph-missing.pdf new file mode 100644 index 000000000..b0af26d9a Binary files /dev/null and b/loop-preflight/testdata/fixtures/font-glyph-missing.pdf differ diff --git a/loop-preflight/testdata/fixtures/manifest.json b/loop-preflight/testdata/fixtures/manifest.json index 503e700f2..c88569170 100644 --- a/loop-preflight/testdata/fixtures/manifest.json +++ b/loop-preflight/testdata/fixtures/manifest.json @@ -872,6 +872,17 @@ "source": "hand-built", "notes": "#668 corpus gap: embedded FontFile2 truncated so inspectPDFFontIntegrity reports TruncatedProgram." }, + { + "id": "font-glyph-missing", + "pdf": "font-glyph-missing.pdf", + "profile": "testdata/profiles/test-font-integrity.json", + "expect": { + "pass": false, + "check_ids": ["font-integrity"] + }, + "source": "hand-built", + "notes": "#114 regression: embedded TrueType subset parses cleanly but has no glyph for shown codes 0xE9 (page content), 0xEA (Form XObject) and 0xEB (Widget appearance stream). Passed clean before glyph coverage was audited." + }, { "id": "thin-parts-clear", "pdf": "thin-parts-clear.pdf", diff --git a/loop-preflight/testdata/snapshots/font-glyph-missing.json b/loop-preflight/testdata/snapshots/font-glyph-missing.json new file mode 100644 index 000000000..566001e62 --- /dev/null +++ b/loop-preflight/testdata/snapshots/font-glyph-missing.json @@ -0,0 +1,52 @@ +{ + "checks": [ + { + "id": "font-integrity", + "status": "failed" + } + ], + "errors": [ + { + "check_id": "font-integrity", + "evidence": { + "code_kind": "character-code", + "defects": [ + "MissingGlyph" + ], + "embedded": true, + "font_resource": "F2+0", + "font_subtype": "4", + "inspection_complete": true, + "missing_codes": [ + 233, + 234, + 235 + ] + }, + "message": "Font 'F2+0' on page 1 has no usable glyph for shown character codes: 233, 234, 235", + "page": 1, + "scope": "page", + "severity": "error", + "type": "font-integrity" + } + ], + "fixups_available": [ + ], + "inspection_complete": true, + "pass": false, + "profile": "Loop Test - Font Integrity", + "schema_kind": "preflight-report", + "schema_version": 4, + "verdict": { + "blocking_finding_ids": [ + "a1974367f8e8fab6" + ], + "reason": "One or more blocking findings require resolution or an active disposition.", + "reason_code": "blocking-findings", + "state": "fail", + "waived_finding_ids": [ + ] + }, + "warnings": [ + ] +} diff --git a/loop-preflight/tools/generate_font_glyph_coverage_fixtures.py b/loop-preflight/tools/generate_font_glyph_coverage_fixtures.py new file mode 100644 index 000000000..68dc540a8 --- /dev/null +++ b/loop-preflight/tools/generate_font_glyph_coverage_fixtures.py @@ -0,0 +1,100 @@ +"""Generate the font-glyph-coverage regression fixture (issue #114). + +Standard library only. The base is font-embedded.pdf, whose embedded TrueType +subset parses cleanly but has a (1, 0) cmap that stops at code 127. The page +content is rewritten to also show codes the program has no glyph for. Every +table still parses, so font-integrity passed it clean before glyph coverage was +audited. + +Each shown-text location uses its own missing code, so the reported codes prove +which locations were traversed: 0xE9 in page content, 0xEA in a Form XObject and +0xEB in a Widget annotation appearance stream. +""" + +from __future__ import annotations + +import re +import sys +import zlib +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +DEFAULT_OUT = ROOT / "testdata" / "fixtures" +BASE = DEFAULT_OUT / "font-embedded.pdf" + +# Octal escapes for codes outside the subset's 0..127 cmap. +PAGE_CODE = b"\351" # 0xE9 +FORM_CODE = b"\352" # 0xEA +ANNOTATION_CODE = b"\353" # 0xEB + + +def _objects(data: bytes) -> dict[int, bytes]: + return { + int(m.group(1)): m.group(2) + for m in re.finditer(rb"(?:^|\n)(\d+) 0 obj\n(.*?)\nendobj", data, re.S) + } + + +def _stream(dictionary: bytes, payload: bytes) -> bytes: + body = zlib.compress(payload, 9) + return b"<< " + dictionary + b" /Length %d /Filter /FlateDecode >>\nstream\n" % len(body) + body + b"\nendstream" + + +def build() -> bytes: + objects = _objects(BASE.read_bytes()) + font_obj = objects[6] + if b"/F2+0 7 0 R" not in font_obj: + raise SystemExit("unexpected font-embedded.pdf layout") + + page_content = ( + b"1 0 0 1 0 0 cm 0 g\n" + b"BT 1 0 0 1 40 150 Tm /F2+0 18 Tf (Frisket fixture: embedded font ) Tj (" + PAGE_CODE + b") Tj ET\n" + b"/Fm0 Do\n" + ) + form_content = b"BT 1 0 0 1 40 120 Tm /F2+0 18 Tf (" + FORM_CODE + b") Tj ET\n" + appearance = b"BT 1 0 0 1 2 2 Tm /F2+0 12 Tf (" + ANNOTATION_CODE + b") Tj ET\n" + resources = b"/Resources << /Font 6 0 R >>" + + page = objects[4].replace( + b"/Resources << /Font 6 0 R /ProcSet", + b"/Annots [ 12 0 R ] /Resources << /Font 6 0 R /XObject << /Fm0 11 0 R >> /ProcSet", + ) + if page == objects[4]: + raise SystemExit("could not attach form and annotation to page") + + out: dict[int, bytes] = dict(objects) + out[4] = page + out[5] = _stream(b"", page_content) + out[11] = _stream(b"/Type /XObject /Subtype /Form /BBox [ 0 0 312 312 ] " + resources, form_content) + out[12] = ( + b"<< /Type /Annot /Subtype /Widget /FT /Tx /T (glyph) /Rect [ 40 60 100 80 ] /F 4 " + b"/AP << /N 13 0 R >> >>" + ) + out[13] = _stream(b"/Type /XObject /Subtype /Form /BBox [ 0 0 60 20 ] " + resources, appearance) + + buffer = bytearray(b"%PDF-1.7\n%\xe2\xe3\xcf\xd3\n") + offsets: dict[int, int] = {} + for number in sorted(out): + offsets[number] = len(buffer) + buffer += b"%d 0 obj\n" % number + out[number] + b"\nendobj\n" + xref = len(buffer) + size = max(out) + 1 + buffer += b"xref\n0 %d\n0000000000 65535 f \n" % size + for number in range(1, size): + buffer += b"%010d 00000 n \n" % offsets[number] + buffer += b"trailer\n<< /Size %d /Root 1 0 R /Info 2 0 R /ID [ <66676c7970686d6973736e67> <66676c7970686d6973736e67> ] >>\n" % size + buffer += b"startxref\n%d\n%%%%EOF\n" % xref + return bytes(buffer) + + +def main(argv: list[str]) -> int: + out = Path(argv[1] if len(argv) > 1 else DEFAULT_OUT) + out.mkdir(parents=True, exist_ok=True) + target = out / "font-glyph-missing.pdf" + target.write_bytes(build()) + print(f"wrote {target.name}") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv))