Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions test/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,11 @@ set_target_properties(
uchardet-tests
)

add_executable(uchardet-lifecycle uchardet-lifecycle.cpp)
target_link_libraries(uchardet-lifecycle ${UCHARDET_LIBRARY})
add_test(NAME native-lifecycle COMMAND uchardet-lifecycle)
set_tests_properties(native-lifecycle PROPERTIES TIMEOUT 10)

# Iterate through all langs.
file(GLOB dirs "[a-z][a-z]")
foreach(dir ${dirs})
Expand Down
29 changes: 29 additions & 0 deletions test/LIFECYCLE.ja.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
<!-- SPDX-License-Identifier: MIT -->
# 小規模なnative lifecycle比較

`uchardet-lifecycle`は通常の空入力、ASCII、UTF-8、cp1252の小入力について、
fresh detectorとreset再利用の同一feed結果を比較するCTest。
4入力を3巡し、直前の文書とは異なるencodingや空入力へ切り替える。

- finalize後のresetがfreshと同じ初期状態へ戻ること。
- 途中までfeedした文書をresetで破棄した場合もfreshと一致すること。
- 同一feed後、および1回のfinalize後のdoneと候補一覧の一致。
- 候補数・順序・encoding・language(nullと空文字も区別)・confidence bit列を比較。
- finalize後のgetter再読で出力が変わらないこと。

独自のencoding正解をこのtestへ埋め込まず、同一入力のfresh結果を対照にする。
自然文corpusのaccuracy試験やchunk間一致試験を置き換えない。
`uchardet_is_done`は「追加入力が不要」の意味であり、Python wrapperの`closed`と
同じ状態を表すと仮定しない。

```sh
cmake -S . -B /disk/lifecycle-build -DBUILD_TESTING=ON -DBUILD_SHARED_LIBS=OFF
cmake --build /disk/lifecycle-build --target uchardet-lifecycle
ctest --test-dir /disk/lifecycle-build -R '^native-lifecycle$' --output-on-failure
```

CTest timeoutは10秒。標準engineや公開APIを変更しない。
error/OOM注入、arbitrary-byte fuzz、大入力、並列利用、language weightの全契約は未対象。
nativeの繰返しfinalizeやfinalize後feedについては、Python wrapperが呼出しを抑止する
挙動とnative C APIの保証を混同せず、今回のtestで保証を追加しない。
P01を再開したり、#117/#118/#119の全完了を主張したりするものではない。
106 changes: 106 additions & 0 deletions test/uchardet-lifecycle.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
// SPDX-License-Identifier: MIT
// Small valid-input lifecycle checks, not arbitrary-byte or allocation-failure tests.
#include "uchardet.h"
#include <cstdint>
#include <cstring>
#include <iostream>
#include <stdexcept>
#include <string>
#include <vector>

class Detector {
public:
Detector() : handle_(uchardet_new()) {
if (!handle_) throw std::runtime_error("detector allocation failed");
}
~Detector() { uchardet_delete(handle_); }
Detector(const Detector&) = delete;
Detector& operator=(const Detector&) = delete;
uchardet_t get() const { return handle_; }
private:
uchardet_t handle_;
};

struct Candidate {
std::string encoding, language;
bool language_present;
std::uint32_t confidence;
bool operator==(const Candidate& other) const {
return encoding == other.encoding && language == other.language &&
language_present == other.language_present && confidence == other.confidence;
}
};

struct Snapshot {
bool done;
std::vector<Candidate> candidates;
bool operator==(const Snapshot& other) const {
return done == other.done && candidates == other.candidates;
}
};

static Snapshot snapshot(const Detector& detector) {
Snapshot value;
value.done = uchardet_is_done(detector.get()) != 0;
const size_t count = uchardet_get_n_candidates(detector.get());
for (size_t i = 0; i < count; ++i) {
Candidate candidate;
const char* encoding = uchardet_get_encoding(detector.get(), i);
const char* language = uchardet_get_language(detector.get(), i);
if (!encoding) throw std::runtime_error("candidate encoding is null");
candidate.encoding = encoding;
candidate.language_present = language != nullptr;
candidate.language = language ? language : "";
const float confidence = uchardet_get_confidence(detector.get(), i);
static_assert(sizeof(confidence) == sizeof(candidate.confidence), "binary32 required");
std::memcpy(&candidate.confidence, &confidence, sizeof(confidence));
value.candidates.push_back(candidate);
}
return value;
}

static void require_equal(const Snapshot& actual, const Snapshot& expected, const char* stage) {
if (!(actual == expected)) throw std::runtime_error(stage);
}

static void feed(const Detector& detector, const std::string& data) {
if (uchardet_handle_data(detector.get(), data.data(), data.size()) != 0)
throw std::runtime_error("normal input failed");
}

int main() {
try {
const std::vector<std::string> inputs = {
"", "plain ASCII text", "caf\xc3\xa9 et th\xc3\xa9", "caf\xe9 et th\xe9"
};
Detector reused;
Detector pristine;
const Snapshot initial = snapshot(pristine);
unsigned comparisons = 0;
for (unsigned cycle = 0; cycle < 3; ++cycle) {
for (const std::string& input : inputs) {
uchardet_reset(reused.get());
require_equal(snapshot(reused), initial, "reset differs from fresh detector");
// Reset must discard a previous, unfinished document as well.
feed(reused, "old ASCII document");
uchardet_reset(reused.get());
require_equal(snapshot(reused), initial, "reset after feed differs from fresh detector");
Detector fresh;
feed(reused, input);
feed(fresh, input);
require_equal(snapshot(reused), snapshot(fresh), "same-feed state differs after reset");
uchardet_data_end(reused.get());
uchardet_data_end(fresh.get());
const Snapshot expected = snapshot(fresh);
require_equal(snapshot(reused), expected, "final candidates differ after reset");
require_equal(snapshot(reused), expected, "candidate getters changed finalized output");
++comparisons;
}
}
std::cout << "fresh/reuse lifecycle comparisons: " << comparisons << '\n';
return 0;
} catch (const std::exception& error) {
std::cerr << error.what() << '\n';
return 1;
}
}
Loading