diff --git a/.gitignore b/.gitignore index c18dd8d..6fe4324 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,4 @@ __pycache__/ +/build/ +/.vscode/ +.DS_Store diff --git a/CMakeLists.txt b/CMakeLists.txt index e9a699e..bf68ec0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,6 @@ ######## Project settings cmake_minimum_required(VERSION 2.8.5) +include(CheckCCompilerFlag) set (PACKAGE_NAME uchardet) project (${PACKAGE_NAME} CXX C) enable_testing() @@ -19,13 +20,8 @@ set ( UCHARDET_VERSION ${UCHARDET_VERSION_MAJOR}.${UCHARDET_VERSION_MINOR}.${UCHARDET_VERSION_REVISION}${version_suffix} ) - -######## Windows - -#if (WIN32) -# set(CMAKE_SHARED_LIBRARY_PREFIX ${CMAKE_INSTALL_PREFIX}) -# set(CMAKE_STATIC_LIBRARY_PREFIX ${CMAKE_INSTALL_PREFIX}) -#endif (WIN32) +set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_STANDARD_REQUIRED ON) ######## Directory @@ -33,13 +29,31 @@ include(GNUInstallDirs) ######## Configuration -option(BUILD_BINARY "Build executable" ON) -option(BUILD_SHARED_LIBS "Build shared library and link executable to it" ON) +option(BUILD_BINARY "Build the CLI tool." ON) +option(BUILD_SHARED_LIBS "Build shared library and link executable to it." ON) +option(CHECK_SSE2 "Check and enable SSE2 extensions if supported. Disabling SSE on platforms which support it may decrease performances." ON) +set(TARGET_ARCHITECTURE "" CACHE STRING "Target CPU architecture. It is autodetected if not specified.") if (BUILD_SHARED_LIBS) option(BUILD_STATIC "Build static library" ON) endif (BUILD_SHARED_LIBS) +if (TARGET_ARCHITECTURE STREQUAL "") + string(TOLOWER ${CMAKE_SYSTEM_PROCESSOR} TARGET_ARCHITECTURE) +endif (TARGET_ARCHITECTURE STREQUAL "") + +if (TARGET_ARCHITECTURE MATCHES ".*(x86)|(amd).*") + CHECK_C_COMPILER_FLAG(-msse2 SUPPORTS_CFLAG_SSE2) + CHECK_C_COMPILER_FLAG(-mfpmath=sse SUPPORTS_CFLAG_SSE_MATH) + if (CHECK_SSE2 AND SUPPORTS_CFLAG_SSE2 AND SUPPORTS_CFLAG_SSE_MATH) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -msse2 -mfpmath=sse") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse2 -mfpmath=sse") + else (CHECK_SSE2 AND SUPPORTS_CFLAG_SSE2 AND SUPPORTS_CFLAG_SSE_MATH) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -ffloat-store") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ffloat-store") + endif (CHECK_SSE2 AND SUPPORTS_CFLAG_SSE2 AND SUPPORTS_CFLAG_SSE_MATH) +endif (TARGET_ARCHITECTURE MATCHES ".*(x86)|(amd).*") + configure_file( uchardet.pc.in uchardet.pc diff --git a/INSTALL b/INSTALL index 60b8234..9b058aa 100644 --- a/INSTALL +++ b/INSTALL @@ -1,4 +1,4 @@ -# INSTALL uchardet +# Building uchardet (generic) `uchardet` uses a typical cmake installation. @@ -14,3 +14,13 @@ Alternatively, use `ccmake`, curses interface to `cmake`. * Install with `make install`. Read `README` for more details on uchardet. + +# Building uchardet on Windows + +The above procedure is generic, which means it should work on any platform. +In particular, it works well on Linux. + +The procedure is the same on Windows, but if you want more details (for +instance which tools to use in order to run CMake on Windows, compiler +information, etc.), the following link may be useful: +https://github.com/BYVoid/uchardet/issues/39#issuecomment-353873891 diff --git a/README.md b/README.md index daaf52c..d845a02 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # uchardet +Forked from [freedesktop/uchardet](https://github.com/freedesktop/uchardet) + [uchardet](https://www.freedesktop.org/wiki/Software/uchardet/) is an encoding detector library, which takes a sequence of bytes in an unknown character encoding without any additional information, and attempts to determine the encoding of the text. Returned encoding names are [iconv](https://www.gnu.org/software/libiconv/)-compatible. uchardet started as a C language binding of the original C++ implementation of the universal charset detection library by Mozilla. It can now detect more charsets, and more reliably than the original implementation. @@ -32,12 +34,12 @@ Techniques used by universalchardet are described at http://www.mozilla.org/proj * ISO-8859-16 * Windows-1250 * IBM852 - * MAC-CENTRALEUROPE + * MacCentralEurope * Czech * Windows-1250 * ISO-8859-2 * IBM852 - * MAC-CENTRALEUROPE + * MacCentralEurope * Danish * ISO-8859-1 * ISO-8859-15 @@ -109,7 +111,7 @@ Techniques used by universalchardet are described at http://www.mozilla.org/proj * ISO-8859-16 * Windows-1250 * IBM852 - * MAC-CENTRALEUROPE + * MacCentralEurope * Portuguese * ISO-8859-1 * ISO-8859-9 @@ -131,13 +133,13 @@ Techniques used by universalchardet are described at http://www.mozilla.org/proj * Windows-1250 * ISO-8859-2 * IBM852 - * MAC-CENTRALEUROPE + * MacCentralEurope * Slovene * ISO-8859-2 * ISO-8859-16 * Windows-1250 * IBM852 - * MAC-CENTRALEUROPE + * MacCentralEurope * Spanish * ISO-8859-1 * ISO-8859-15 @@ -162,57 +164,19 @@ Techniques used by universalchardet are described at http://www.mozilla.org/proj ## Installation -### Debian/Ubuntu/Mint - - apt-get install uchardet libuchardet-dev - -### Mageia - - urpmi libuchardet libuchardet-devel - -### Fedora - - dnf install uchardet uchardet-devel - -### Mac - - brew install uchardet - ### Build from source -Releases are available from: -https://www.freedesktop.org/software/uchardet/releases/ - If you prefer a development version, clone the git repository: - git clone git://anongit.freedesktop.org/uchardet/uchardet + git clone https://github.com/PyYoshi/uchardet.git -The source can be browsed at: https://cgit.freedesktop.org/uchardet/uchardet/ +The source can be browsed at: https://github.com/PyYoshi/uchardet - cmake . + mkdir build/ && cd build/ + cmake .. make make install -### Build with flatpak-builder - -Here is a working "module" section to include in your Flatpak's json manifest: - -``` -"modules": [ - { - "name": "uchardet", - "buildsystem": "cmake", - "builddir": true, - "config-opts": [ "-DCMAKE_INSTALL_LIBDIR=lib" ], - "sources": [ - { - ... - } - ] - } -] -``` - ## Usage ### Command Line @@ -233,19 +197,7 @@ Options: ``` ### Library -See [uchardet.h](https://cgit.freedesktop.org/uchardet/uchardet/tree/src/uchardet.h) - -## Related Projects - - * [python-chardet](https://github.com/chardet/chardet) Python port - * [ruby-rchardet](http://rubyforge.org/projects/chardet/) Ruby port - * [juniversalchardet](http://code.google.com/p/juniversalchardet/) Java port of universalchardet - * [jchardet](http://jchardet.sourceforge.net/) Java port of chardet - * [nuniversalchardet](http://code.google.com/p/nuniversalchardet/) C# port of universalchardet - * [nchardet](http://www.conceptdevelopment.net/Localization/NCharDet/) C# port of chardet - * [uchardet-enhanced](https://bitbucket.org/medoc/uchardet-enhanced) A fork of mozilla universalchardet - * [rust-uchardet](https://github.com/emk/rust-uchardet) Rust language binding of uchardet - * [libchardet](https://ftp.oops.org/pub/oops/libchardet/) Another C/C++ API wrapping Mozilla code. +See [uchardet.h](https://github.com/PyYoshi/uchardet/blob/cchardet/src/uchardet.h) ## Licenses diff --git a/script/BuildLangModelLogs/LangEstonianModel.log b/script/BuildLangModelLogs/LangEstonianModel.log index f1095eb..f11652c 100644 --- a/script/BuildLangModelLogs/LangEstonianModel.log +++ b/script/BuildLangModelLogs/LangEstonianModel.log @@ -156,4 +156,4 @@ First 512 (typical positive ratio): 0.9972721312183132 Next 512 (512-1024): 9.687262863877811e-05 Rest: -5.204170427930421e-18 -- Processing end: 2016-09-26 23:47:54.561846 +- Processing end: 2016-09-26 23:47:54.561846 \ No newline at end of file diff --git a/script/charsets/mac-centraleurope.py b/script/charsets/mac-centraleurope.py index bf201e6..87cb9c3 100644 --- a/script/charsets/mac-centraleurope.py +++ b/script/charsets/mac-centraleurope.py @@ -40,7 +40,7 @@ from codepoints import * -name = 'MAC-CENTRALEUROPE' +name = 'MacCentralEurope' aliases = [] language = \ diff --git a/script/gen.sh b/script/gen.sh new file mode 100755 index 0000000..27737b3 --- /dev/null +++ b/script/gen.sh @@ -0,0 +1,27 @@ +#!/bin/bash +python ./BuildLangModel.py ar --max-page=100 --max-depth=4 +python ./BuildLangModel.py cs --max-page=100 --max-depth=4 +python ./BuildLangModel.py da --max-page=100 --max-depth=4 +python ./BuildLangModel.py de --max-page=100 --max-depth=4 +python ./BuildLangModel.py el --max-page=100 --max-depth=4 +python ./BuildLangModel.py eo --max-page=100 --max-depth=4 +python ./BuildLangModel.py es --max-page=100 --max-depth=4 +python ./BuildLangModel.py et --max-page=100 --max-depth=4 +python ./BuildLangModel.py fi --max-page=100 --max-depth=4 +python ./BuildLangModel.py fr --max-page=100 --max-depth=4 +python ./BuildLangModel.py ga --max-page=100 --max-depth=4 +python ./BuildLangModel.py hr --max-page=100 --max-depth=4 +python ./BuildLangModel.py hu --max-page=100 --max-depth=4 +python ./BuildLangModel.py it --max-page=100 --max-depth=4 +python ./BuildLangModel.py lt --max-page=100 --max-depth=4 +python ./BuildLangModel.py lv --max-page=100 --max-depth=4 +python ./BuildLangModel.py mt --max-page=100 --max-depth=4 +python ./BuildLangModel.py pl --max-page=100 --max-depth=4 +python ./BuildLangModel.py pt --max-page=100 --max-depth=4 +python ./BuildLangModel.py ro --max-page=100 --max-depth=4 +python ./BuildLangModel.py sk --max-page=100 --max-depth=4 +python ./BuildLangModel.py sl --max-page=100 --max-depth=4 +python ./BuildLangModel.py sv --max-page=100 --max-depth=4 +python ./BuildLangModel.py th --max-page=100 --max-depth=4 +python ./BuildLangModel.py tr --max-page=100 --max-depth=4 +python ./BuildLangModel.py vi --max-page=100 --max-depth=4 diff --git a/script/langs/cs.py b/script/langs/cs.py index 009ceec..f6865ed 100644 --- a/script/langs/cs.py +++ b/script/langs/cs.py @@ -50,7 +50,7 @@ # ASCII characters are also used in French. use_ascii = True # The charsets we want to support and create data for. -charsets = ['ISO-8859-2', 'Windows-1250', 'IBM852', 'MAC-CENTRALEUROPE'] +charsets = ['ISO-8859-2', 'Windows-1250', 'IBM852', 'MacCentralEurope'] ## Optional Properties ## diff --git a/script/langs/hr.py b/script/langs/hr.py index 3b0dea5..12d9388 100644 --- a/script/langs/hr.py +++ b/script/langs/hr.py @@ -46,7 +46,7 @@ code = 'hr' use_ascii = True charsets = ['ISO-8859-2', 'ISO-8859-13', 'ISO-8859-16', - 'Windows-1250', 'IBM852', 'MAC-CENTRALEUROPE'] + 'Windows-1250', 'IBM852', 'MacCentralEurope'] ## Optional Properties ## diff --git a/script/langs/pl.py b/script/langs/pl.py index 54485a1..7dd1ffb 100644 --- a/script/langs/pl.py +++ b/script/langs/pl.py @@ -51,7 +51,7 @@ use_ascii = True # The charsets we want to support and create data for. charsets = ['ISO-8859-2', 'ISO-8859-13', 'ISO-8859-16', - 'Windows-1250', 'IBM852', 'MAC-CENTRALEUROPE'] + 'Windows-1250', 'IBM852', 'MacCentralEurope'] ## Optional Properties ## diff --git a/script/langs/sk.py b/script/langs/sk.py index c670839..d1e4689 100644 --- a/script/langs/sk.py +++ b/script/langs/sk.py @@ -50,7 +50,7 @@ # ASCII characters are also used in French. use_ascii = True # The charsets we want to support and create data for. -charsets = ['ISO-8859-2', 'Windows-1250', 'IBM852', 'MAC-CENTRALEUROPE'] +charsets = ['ISO-8859-2', 'Windows-1250', 'IBM852', 'MacCentralEurope'] ## Optional Properties ## diff --git a/script/langs/sl.py b/script/langs/sl.py index bf02bf8..5ffe69d 100644 --- a/script/langs/sl.py +++ b/script/langs/sl.py @@ -46,7 +46,7 @@ code = 'sl' use_ascii = True charsets = ['ISO-8859-2', 'ISO-8859-16', - 'Windows-1250', 'IBM852', 'MAC-CENTRALEUROPE'] + 'Windows-1250', 'IBM852', 'MacCentralEurope'] ## Optional Properties ## diff --git a/script/requirements.txt b/script/requirements.txt new file mode 100644 index 0000000..840e6bc --- /dev/null +++ b/script/requirements.txt @@ -0,0 +1,2 @@ +requests==2.31.0 +wikipedia==1.4.0 \ No newline at end of file diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 952b594..a1db01a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -72,7 +72,13 @@ if (CMAKE_BUILD_TYPE MATCHES Debug) add_definitions( -O0 -g3 + -fsanitize=address ) + set(CMAKE_CXX_FLAGS "-fsanitize=address") + set(CMAKE_C_FLAGS "-fsanitize=address") + set(CMAKE_EXE_LINKER_FLAGS "-fsanitize=address") + set(CMAKE_MODULE_LINKER_FLAGS "-fsanitize=address") + set(CMAKE_SHARED_LINKER_FLAGS "-fsanitize=address") endif (CMAKE_BUILD_TYPE MATCHES Debug) add_library( @@ -152,4 +158,9 @@ include(symbols.cmake) if (BUILD_BINARY) add_subdirectory(tools) + add_subdirectory(fuzzing) endif (BUILD_BINARY) + +if (BUILD_FUZZER) + add_subdirectory(fuzzing) +endif (BUILD_FUZZER) diff --git a/src/EUCTWFreq.tab b/src/EUCTWFreq.tab index 3a189ae..39f9d97 100644 --- a/src/EUCTWFreq.tab +++ b/src/EUCTWFreq.tab @@ -57,7 +57,7 @@ #define EUCTW_TYPICAL_DISTRIBUTION_RATIO (float)0.75 //Char to FreqOrder table , -#define EUCTW_TABLE_SIZE 8102 +#define EUCTW_TABLE_SIZE 5376 static const PRInt16 EUCTWCharToFreqOrder[] = { diff --git a/src/LangModels/LangCroatianModel.cpp b/src/LangModels/LangCroatianModel.cpp index 58f882e..7151522 100644 --- a/src/LangModels/LangCroatianModel.cpp +++ b/src/LangModels/LangCroatianModel.cpp @@ -124,7 +124,7 @@ static const unsigned char Iso_8859_16_CharToOrderMap[] = }; /*X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 XA XB XC XD XE XF */ -static const unsigned char Mac_Centraleurope_CharToOrderMap[] = +static const unsigned char Maccentraleurope_CharToOrderMap[] = { CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,RET,CTR,CTR,RET,CTR,CTR, /* 0X */ CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR, /* 1X */ @@ -261,14 +261,14 @@ const SequenceModel Iso_8859_16CroatianModel = "ISO-8859-16" }; -const SequenceModel Mac_CentraleuropeCroatianModel = +const SequenceModel MaccentraleuropeCroatianModel = { - Mac_Centraleurope_CharToOrderMap, + Maccentraleurope_CharToOrderMap, CroatianLangModel, 31, (float)0.9989731099787131, PR_TRUE, - "MAC-CENTRALEUROPE" + "MacCentralEurope" }; const SequenceModel Iso_8859_13CroatianModel = diff --git a/src/LangModels/LangCzechModel.cpp b/src/LangModels/LangCzechModel.cpp index 4e74ed2..0b8b5bd 100644 --- a/src/LangModels/LangCzechModel.cpp +++ b/src/LangModels/LangCzechModel.cpp @@ -82,7 +82,7 @@ static const unsigned char Windows_1250_CharToOrderMap[] = }; /*X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 XA XB XC XD XE XF */ -static const unsigned char Mac_Centraleurope_CharToOrderMap[] = +static const unsigned char Maccentraleurope_CharToOrderMap[] = { CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,RET,CTR,CTR,RET,CTR,CTR, /* 0X */ CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR, /* 1X */ @@ -250,14 +250,14 @@ const SequenceModel Windows_1250CzechModel = "WINDOWS-1250" }; -const SequenceModel Mac_CentraleuropeCzechModel = +const SequenceModel MaccentraleuropeCzechModel = { - Mac_Centraleurope_CharToOrderMap, + Maccentraleurope_CharToOrderMap, CzechLangModel, 41, (float)0.9786035192432675, PR_TRUE, - "MAC-CENTRALEUROPE" + "MacCentralEurope" }; const SequenceModel Ibm852CzechModel = @@ -278,4 +278,4 @@ const SequenceModel Iso_8859_2CzechModel = (float)0.9786035192432675, PR_TRUE, "ISO-8859-2" -}; \ No newline at end of file +}; diff --git a/src/LangModels/LangEstonianModel.cpp b/src/LangModels/LangEstonianModel.cpp index c5fa9b3..362f280 100644 --- a/src/LangModels/LangEstonianModel.cpp +++ b/src/LangModels/LangEstonianModel.cpp @@ -260,4 +260,4 @@ const SequenceModel Windows_1257EstonianModel = (float)0.9972721312183132, PR_TRUE, "WINDOWS-1257" -}; +}; \ No newline at end of file diff --git a/src/LangModels/LangPolishModel.cpp b/src/LangModels/LangPolishModel.cpp index e7379d1..01370c9 100644 --- a/src/LangModels/LangPolishModel.cpp +++ b/src/LangModels/LangPolishModel.cpp @@ -124,7 +124,7 @@ static const unsigned char Iso_8859_2_CharToOrderMap[] = }; /*X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 XA XB XC XD XE XF */ -static const unsigned char Mac_Centraleurope_CharToOrderMap[] = +static const unsigned char Maccentraleurope_CharToOrderMap[] = { CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,RET,CTR,CTR,RET,CTR,CTR, /* 0X */ CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR, /* 1X */ @@ -267,14 +267,14 @@ const SequenceModel Iso_8859_2PolishModel = "ISO-8859-2" }; -const SequenceModel Mac_CentraleuropePolishModel = +const SequenceModel MaccentraleuropePolishModel = { - Mac_Centraleurope_CharToOrderMap, + Maccentraleurope_CharToOrderMap, PolishLangModel, 37, (float)0.9894531815946438, PR_TRUE, - "MAC-CENTRALEUROPE" + "MacCentralEurope" }; const SequenceModel Iso_8859_13PolishModel = @@ -295,4 +295,4 @@ const SequenceModel Windows_1250PolishModel = (float)0.9894531815946438, PR_TRUE, "WINDOWS-1250" -}; \ No newline at end of file +}; diff --git a/src/LangModels/LangRussianModel.cpp b/src/LangModels/LangRussianModel.cpp index a532049..e21bfd1 100644 --- a/src/LangModels/LangRussianModel.cpp +++ b/src/LangModels/LangRussianModel.cpp @@ -337,7 +337,7 @@ const SequenceModel MacCyrillicRussianModel = 64, (float)0.976601, PR_FALSE, - "MAC-CYRILLIC" + "MacCyrillic" }; const SequenceModel Ibm866RussianModel = diff --git a/src/LangModels/LangSlovakModel.cpp b/src/LangModels/LangSlovakModel.cpp index cfa94aa..56cee68 100644 --- a/src/LangModels/LangSlovakModel.cpp +++ b/src/LangModels/LangSlovakModel.cpp @@ -103,7 +103,7 @@ static const unsigned char Iso_8859_2_CharToOrderMap[] = }; /*X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 XA XB XC XD XE XF */ -static const unsigned char Mac_Centraleurope_CharToOrderMap[] = +static const unsigned char Maccentraleurope_CharToOrderMap[] = { CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,RET,CTR,CTR,RET,CTR,CTR, /* 0X */ CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR, /* 1X */ @@ -268,14 +268,14 @@ const SequenceModel Iso_8859_2SlovakModel = "ISO-8859-2" }; -const SequenceModel Mac_CentraleuropeSlovakModel = +const SequenceModel MaccentraleuropeSlovakModel = { - Mac_Centraleurope_CharToOrderMap, + Maccentraleurope_CharToOrderMap, SlovakLangModel, 45, (float)0.9733303573968434, PR_TRUE, - "MAC-CENTRALEUROPE" + "MacCentralEurope" }; const SequenceModel Windows_1250SlovakModel = diff --git a/src/LangModels/LangSloveneModel.cpp b/src/LangModels/LangSloveneModel.cpp index da28d86..923fcec 100644 --- a/src/LangModels/LangSloveneModel.cpp +++ b/src/LangModels/LangSloveneModel.cpp @@ -124,7 +124,7 @@ static const unsigned char Windows_1250_CharToOrderMap[] = }; /*X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 XA XB XC XD XE XF */ -static const unsigned char Mac_Centraleurope_CharToOrderMap[] = +static const unsigned char Maccentraleurope_CharToOrderMap[] = { CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,RET,CTR,CTR,RET,CTR,CTR, /* 0X */ CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR, /* 1X */ @@ -238,14 +238,14 @@ const SequenceModel Windows_1250SloveneModel = "WINDOWS-1250" }; -const SequenceModel Mac_CentraleuropeSloveneModel = +const SequenceModel MaccentraleuropeSloveneModel = { - Mac_Centraleurope_CharToOrderMap, + Maccentraleurope_CharToOrderMap, SloveneLangModel, 29, (float)0.9983524317161332, PR_TRUE, - "MAC-CENTRALEUROPE" + "MacCentralEurope" }; const SequenceModel Ibm852SloveneModel = diff --git a/src/fuzzing/CMakeLists.txt b/src/fuzzing/CMakeLists.txt new file mode 100644 index 0000000..c707ad4 --- /dev/null +++ b/src/fuzzing/CMakeLists.txt @@ -0,0 +1,36 @@ +# for macOS: $ cmake -DCMAKE_CXX_COMPILER="`brew --prefix llvm`/bin/clang++" ../ + +set( + UCHARDET_SOURCES + uchardet_fuzzer.cpp +) + +add_definitions( + -O0 + -g3 + -fsanitize=fuzzer,address +) +set(CMAKE_CXX_FLAGS "-fsanitize=fuzzer,address") +set(CMAKE_C_FLAGS "-fsanitize=fuzzer,address") +set(CMAKE_EXE_LINKER_FLAGS "-fsanitize=fuzzer,address") +set(CMAKE_MODULE_LINKER_FLAGS "-fsanitize=fuzzer,address") +set(CMAKE_SHARED_LINKER_FLAGS "-fsanitize=fuzzer,address") + +set(FUZZER_BINARY uchardet_fuzzer) + +add_executable( + ${FUZZER_BINARY} + ${UCHARDET_SOURCES} +) + +target_link_libraries( + ${FUZZER_BINARY} + ${UCHARDET_LIBRARY} +) + +# install( +# TARGETS +# ${FUZZER_BINARY} +# RUNTIME DESTINATION +# ${CMAKE_INSTALL_BINDIR} +# ) diff --git a/src/fuzzing/uchardet_fuzzer.cpp b/src/fuzzing/uchardet_fuzzer.cpp new file mode 100644 index 0000000..21da646 --- /dev/null +++ b/src/fuzzing/uchardet_fuzzer.cpp @@ -0,0 +1,64 @@ +/* ***** BEGIN LICENSE BLOCK ***** + * Version: MPL 1.1/GPL 2.0/LGPL 2.1 + * + * The contents of this file are subject to the Mozilla Public License Version + * 1.1 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.mozilla.org/MPL/ + * + * Software distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License + * for the specific language governing rights and limitations under the + * License. + * + * The Original Code is Mozilla Universal charset detector code. + * + * The Initial Developer of the Original Code is + * Netscape Communications Corporation. + * Portions created by the Initial Developer are Copyright (C) 2001 + * the Initial Developer. All Rights Reserved. + * + * Contributor(s): + * PyYoshi + * + * Alternatively, the contents of this file may be used under the terms of + * either the GNU General Public License Version 2 or later (the "GPL"), or + * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), + * in which case the provisions of the GPL or the LGPL are applicable instead + * of those above. If you wish to allow use of your version of this file only + * under the terms of either the GPL or the LGPL, and not to allow others to + * use your version of this file under the terms of the MPL, indicate your + * decision by deleting the provisions above and replace them with the notice + * and other provisions required by the GPL or the LGPL. If you do not delete + * the provisions above, a recipient may use your version of this file under + * the terms of any one of the MPL, the GPL or the LGPL. + * + * ***** END LICENSE BLOCK ***** */ +#include "../uchardet.h" +#include +#include +#include +#include + +// extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size); + +extern "C" +int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { + const char* buff = reinterpret_cast(data); + + uchardet_t handle = uchardet_new(); + + int retval = uchardet_handle_data(handle, buff, size); + if (retval == HANDLE_DATA_RESULT_ERROR) + { + fprintf(stderr, "Handle data error.\n"); + exit(1); + } + + uchardet_data_end(handle); + uchardet_get_charset(handle); + uchardet_get_confidence(handle); + uchardet_delete(handle); + + return 0; +} diff --git a/src/nsBig5Prober.cpp b/src/nsBig5Prober.cpp index 7a85abb..3e47d6e 100644 --- a/src/nsBig5Prober.cpp +++ b/src/nsBig5Prober.cpp @@ -46,7 +46,7 @@ void nsBig5Prober::Reset(void) nsProbingState nsBig5Prober::HandleData(const char* aBuf, PRUint32 aLen) { - nsSMState codingState; + PRUint32 codingState; for (PRUint32 i = 0; i < aLen; i++) { diff --git a/src/nsCharSetProber.cpp b/src/nsCharSetProber.cpp index 0429dd1..79d5ace 100644 --- a/src/nsCharSetProber.cpp +++ b/src/nsCharSetProber.cpp @@ -74,7 +74,7 @@ PRBool nsCharSetProber::FilterWithoutEnglishLetters(const char* aBuf, PRUint32 a if (meetMSB && curPtr > prevPtr) while (prevPtr < curPtr) *newptr++ = *prevPtr++; - newLen = newptr - *newBuf; + newLen = (PRUint32) (newptr - *newBuf); return PR_TRUE; } @@ -119,7 +119,7 @@ PRBool nsCharSetProber::FilterWithEnglishLetters(const char* aBuf, PRUint32 aLen while (prevPtr < curPtr) *newptr++ = *prevPtr++; - newLen = newptr - *newBuf; + newLen = (PRUint32) (newptr - *newBuf); return PR_TRUE; } diff --git a/src/nsCodingStateMachine.h b/src/nsCodingStateMachine.h index 819f9ab..8861118 100644 --- a/src/nsCodingStateMachine.h +++ b/src/nsCodingStateMachine.h @@ -39,11 +39,12 @@ #include "nsPkgInt.h" -typedef enum { - eStart = 0, - eError = 1, - eItsMe = 2 -} nsSMState; +/* Apart from these 3 generic states, machine states are specific to + * each charset prober. + */ +#define eStart 0 +#define eError 1 +#define eItsMe 2 #define GETCLASS(c) GETFROMPCK(((unsigned char)(c)), mModel->classTable) @@ -60,7 +61,7 @@ typedef struct class nsCodingStateMachine { public: nsCodingStateMachine(const SMModel* sm) : mModel(sm) { mCurrentState = eStart; } - nsSMState NextState(char c){ + PRUint32 NextState(char c){ //for each byte we get its class , if it is first byte, we also get byte length PRUint32 byteCls = GETCLASS(c); if (mCurrentState == eStart) @@ -69,8 +70,8 @@ class nsCodingStateMachine { mCurrentCharLen = mModel->charLenTable[byteCls]; } //from byte's class and stateTable, we get its next state - mCurrentState=(nsSMState)GETFROMPCK(mCurrentState*(mModel->classFactor)+byteCls, - mModel->stateTable); + mCurrentState = GETFROMPCK(mCurrentState * mModel->classFactor + byteCls, + mModel->stateTable); mCurrentBytePos++; return mCurrentState; } @@ -79,7 +80,7 @@ class nsCodingStateMachine { const char * GetCodingStateMachine() {return mModel->name;} protected: - nsSMState mCurrentState; + PRUint32 mCurrentState; PRUint32 mCurrentCharLen; PRUint32 mCurrentBytePos; diff --git a/src/nsEUCJPProber.cpp b/src/nsEUCJPProber.cpp index 54861b3..f84d154 100644 --- a/src/nsEUCJPProber.cpp +++ b/src/nsEUCJPProber.cpp @@ -52,7 +52,7 @@ void nsEUCJPProber::Reset(void) nsProbingState nsEUCJPProber::HandleData(const char* aBuf, PRUint32 aLen) { - nsSMState codingState; + PRUint32 codingState; for (PRUint32 i = 0; i < aLen; i++) { diff --git a/src/nsEUCKRProber.cpp b/src/nsEUCKRProber.cpp index 3632f1f..6aae8ae 100644 --- a/src/nsEUCKRProber.cpp +++ b/src/nsEUCKRProber.cpp @@ -47,7 +47,7 @@ void nsEUCKRProber::Reset(void) nsProbingState nsEUCKRProber::HandleData(const char* aBuf, PRUint32 aLen) { - nsSMState codingState; + PRUint32 codingState; for (PRUint32 i = 0; i < aLen; i++) { diff --git a/src/nsEUCTWProber.cpp b/src/nsEUCTWProber.cpp index a06e074..7e61ea1 100644 --- a/src/nsEUCTWProber.cpp +++ b/src/nsEUCTWProber.cpp @@ -47,7 +47,7 @@ void nsEUCTWProber::Reset(void) nsProbingState nsEUCTWProber::HandleData(const char* aBuf, PRUint32 aLen) { - nsSMState codingState; + PRUint32 codingState; for (PRUint32 i = 0; i < aLen; i++) { diff --git a/src/nsEscCharsetProber.cpp b/src/nsEscCharsetProber.cpp index 464c753..d093ee4 100644 --- a/src/nsEscCharsetProber.cpp +++ b/src/nsEscCharsetProber.cpp @@ -75,7 +75,7 @@ void nsEscCharSetProber::Reset(void) nsProbingState nsEscCharSetProber::HandleData(const char* aBuf, PRUint32 aLen) { - nsSMState codingState; + PRUint32 codingState; PRInt32 j; PRUint32 i; diff --git a/src/nsEscSM.cpp b/src/nsEscSM.cpp index eed1b7c..cad3df8 100644 --- a/src/nsEscSM.cpp +++ b/src/nsEscSM.cpp @@ -197,7 +197,7 @@ PCK4BITS(eError,eError,eError,eItsMe,eError,eError,eError,eError),//38-3f PCK4BITS(eError,eError,eError,eError,eItsMe,eError,eStart,eStart) //40-47 }; -static const PRUint32 ISO2022JPCharLenTable[] = {0, 0, 0, 0, 0, 0, 0, 0}; +static const PRUint32 ISO2022JPCharLenTable[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; const SMModel ISO2022JPSMModel = { {eIdxSft4bits, eSftMsk4bits, eBitSft4bits, eUnitMsk4bits, ISO2022JP_cls }, diff --git a/src/nsGB2312Prober.cpp b/src/nsGB2312Prober.cpp index b6d469c..eac0762 100644 --- a/src/nsGB2312Prober.cpp +++ b/src/nsGB2312Prober.cpp @@ -52,7 +52,7 @@ void nsGB18030Prober::Reset(void) nsProbingState nsGB18030Prober::HandleData(const char* aBuf, PRUint32 aLen) { - nsSMState codingState; + PRUint32 codingState; for (PRUint32 i = 0; i < aLen; i++) { diff --git a/src/nsSBCSGroupProber.cpp b/src/nsSBCSGroupProber.cpp index 66738ea..42b1b34 100644 --- a/src/nsSBCSGroupProber.cpp +++ b/src/nsSBCSGroupProber.cpp @@ -128,19 +128,19 @@ nsSBCSGroupProber::nsSBCSGroupProber() mProbers[46] = new nsSingleByteCharSetProber(&Windows_1250CzechModel); mProbers[47] = new nsSingleByteCharSetProber(&Iso_8859_2CzechModel); - mProbers[48] = new nsSingleByteCharSetProber(&Mac_CentraleuropeCzechModel); + mProbers[48] = new nsSingleByteCharSetProber(&MaccentraleuropeCzechModel); mProbers[49] = new nsSingleByteCharSetProber(&Ibm852CzechModel); mProbers[50] = new nsSingleByteCharSetProber(&Windows_1250SlovakModel); mProbers[51] = new nsSingleByteCharSetProber(&Iso_8859_2SlovakModel); - mProbers[52] = new nsSingleByteCharSetProber(&Mac_CentraleuropeSlovakModel); + mProbers[52] = new nsSingleByteCharSetProber(&MaccentraleuropeSlovakModel); mProbers[53] = new nsSingleByteCharSetProber(&Ibm852SlovakModel); mProbers[54] = new nsSingleByteCharSetProber(&Windows_1250PolishModel); mProbers[55] = new nsSingleByteCharSetProber(&Iso_8859_2PolishModel); mProbers[56] = new nsSingleByteCharSetProber(&Iso_8859_13PolishModel); mProbers[57] = new nsSingleByteCharSetProber(&Iso_8859_16PolishModel); - mProbers[58] = new nsSingleByteCharSetProber(&Mac_CentraleuropePolishModel); + mProbers[58] = new nsSingleByteCharSetProber(&MaccentraleuropePolishModel); mProbers[59] = new nsSingleByteCharSetProber(&Ibm852PolishModel); mProbers[60] = new nsSingleByteCharSetProber(&Iso_8859_1FinnishModel); @@ -150,46 +150,46 @@ nsSBCSGroupProber::nsSBCSGroupProber() mProbers[64] = new nsSingleByteCharSetProber(&Iso_8859_15FinnishModel); mProbers[65] = new nsSingleByteCharSetProber(&Windows_1252FinnishModel); - mProbers[65] = new nsSingleByteCharSetProber(&Iso_8859_1ItalianModel); - mProbers[66] = new nsSingleByteCharSetProber(&Iso_8859_3ItalianModel); - mProbers[67] = new nsSingleByteCharSetProber(&Iso_8859_9ItalianModel); - mProbers[68] = new nsSingleByteCharSetProber(&Iso_8859_15ItalianModel); - mProbers[69] = new nsSingleByteCharSetProber(&Windows_1252ItalianModel); - - mProbers[70] = new nsSingleByteCharSetProber(&Windows_1250CroatianModel); - mProbers[71] = new nsSingleByteCharSetProber(&Iso_8859_2CroatianModel); - mProbers[72] = new nsSingleByteCharSetProber(&Iso_8859_13CroatianModel); - mProbers[73] = new nsSingleByteCharSetProber(&Iso_8859_16CroatianModel); - mProbers[74] = new nsSingleByteCharSetProber(&Mac_CentraleuropeCroatianModel); - mProbers[75] = new nsSingleByteCharSetProber(&Ibm852CroatianModel); - - mProbers[76] = new nsSingleByteCharSetProber(&Windows_1252EstonianModel); - mProbers[77] = new nsSingleByteCharSetProber(&Windows_1257EstonianModel); - mProbers[78] = new nsSingleByteCharSetProber(&Iso_8859_4EstonianModel); - mProbers[79] = new nsSingleByteCharSetProber(&Iso_8859_13EstonianModel); - mProbers[80] = new nsSingleByteCharSetProber(&Iso_8859_15EstonianModel); - - mProbers[81] = new nsSingleByteCharSetProber(&Iso_8859_1IrishModel); - mProbers[82] = new nsSingleByteCharSetProber(&Iso_8859_9IrishModel); - mProbers[83] = new nsSingleByteCharSetProber(&Iso_8859_15IrishModel); - mProbers[84] = new nsSingleByteCharSetProber(&Windows_1252IrishModel); - - mProbers[85] = new nsSingleByteCharSetProber(&Windows_1250RomanianModel); - mProbers[86] = new nsSingleByteCharSetProber(&Iso_8859_2RomanianModel); - mProbers[87] = new nsSingleByteCharSetProber(&Iso_8859_16RomanianModel); - mProbers[88] = new nsSingleByteCharSetProber(&Ibm852RomanianModel); - - mProbers[89] = new nsSingleByteCharSetProber(&Windows_1250SloveneModel); - mProbers[90] = new nsSingleByteCharSetProber(&Iso_8859_2SloveneModel); - mProbers[91] = new nsSingleByteCharSetProber(&Iso_8859_16SloveneModel); - mProbers[92] = new nsSingleByteCharSetProber(&Mac_CentraleuropeSloveneModel); - mProbers[93] = new nsSingleByteCharSetProber(&Ibm852SloveneModel); - - mProbers[94] = new nsSingleByteCharSetProber(&Iso_8859_1SwedishModel); - mProbers[95] = new nsSingleByteCharSetProber(&Iso_8859_4SwedishModel); - mProbers[96] = new nsSingleByteCharSetProber(&Iso_8859_9SwedishModel); - mProbers[97] = new nsSingleByteCharSetProber(&Iso_8859_15SwedishModel); - mProbers[98] = new nsSingleByteCharSetProber(&Windows_1252SwedishModel); + mProbers[66] = new nsSingleByteCharSetProber(&Iso_8859_1ItalianModel); + mProbers[67] = new nsSingleByteCharSetProber(&Iso_8859_3ItalianModel); + mProbers[68] = new nsSingleByteCharSetProber(&Iso_8859_9ItalianModel); + mProbers[69] = new nsSingleByteCharSetProber(&Iso_8859_15ItalianModel); + mProbers[70] = new nsSingleByteCharSetProber(&Windows_1252ItalianModel); + + mProbers[71] = new nsSingleByteCharSetProber(&Windows_1250CroatianModel); + mProbers[72] = new nsSingleByteCharSetProber(&Iso_8859_2CroatianModel); + mProbers[73] = new nsSingleByteCharSetProber(&Iso_8859_13CroatianModel); + mProbers[74] = new nsSingleByteCharSetProber(&Iso_8859_16CroatianModel); + mProbers[75] = new nsSingleByteCharSetProber(&MaccentraleuropeCroatianModel); + mProbers[76] = new nsSingleByteCharSetProber(&Ibm852CroatianModel); + + mProbers[77] = new nsSingleByteCharSetProber(&Windows_1252EstonianModel); + mProbers[78] = new nsSingleByteCharSetProber(&Windows_1257EstonianModel); + mProbers[79] = new nsSingleByteCharSetProber(&Iso_8859_4EstonianModel); + mProbers[80] = new nsSingleByteCharSetProber(&Iso_8859_13EstonianModel); + mProbers[81] = new nsSingleByteCharSetProber(&Iso_8859_15EstonianModel); + + mProbers[82] = new nsSingleByteCharSetProber(&Iso_8859_1IrishModel); + mProbers[83] = new nsSingleByteCharSetProber(&Iso_8859_9IrishModel); + mProbers[84] = new nsSingleByteCharSetProber(&Iso_8859_15IrishModel); + mProbers[85] = new nsSingleByteCharSetProber(&Windows_1252IrishModel); + + mProbers[86] = new nsSingleByteCharSetProber(&Windows_1250RomanianModel); + mProbers[87] = new nsSingleByteCharSetProber(&Iso_8859_2RomanianModel); + mProbers[88] = new nsSingleByteCharSetProber(&Iso_8859_16RomanianModel); + mProbers[89] = new nsSingleByteCharSetProber(&Ibm852RomanianModel); + + mProbers[90] = new nsSingleByteCharSetProber(&Windows_1250SloveneModel); + mProbers[91] = new nsSingleByteCharSetProber(&Iso_8859_2SloveneModel); + mProbers[92] = new nsSingleByteCharSetProber(&Iso_8859_16SloveneModel); + mProbers[93] = new nsSingleByteCharSetProber(&MaccentraleuropeSloveneModel); + mProbers[94] = new nsSingleByteCharSetProber(&Ibm852SloveneModel); + + mProbers[95] = new nsSingleByteCharSetProber(&Iso_8859_1SwedishModel); + mProbers[96] = new nsSingleByteCharSetProber(&Iso_8859_4SwedishModel); + mProbers[97] = new nsSingleByteCharSetProber(&Iso_8859_9SwedishModel); + mProbers[98] = new nsSingleByteCharSetProber(&Iso_8859_15SwedishModel); + mProbers[99] = new nsSingleByteCharSetProber(&Windows_1252SwedishModel); Reset(); } diff --git a/src/nsSBCSGroupProber.h b/src/nsSBCSGroupProber.h index 64c021b..ec72324 100644 --- a/src/nsSBCSGroupProber.h +++ b/src/nsSBCSGroupProber.h @@ -40,7 +40,7 @@ #define nsSBCSGroupProber_h__ -#define NUM_OF_SBCS_PROBERS 99 +#define NUM_OF_SBCS_PROBERS 100 class nsCharSetProber; class nsSBCSGroupProber: public nsCharSetProber { diff --git a/src/nsSBCharSetProber.cpp b/src/nsSBCharSetProber.cpp index 1f7f473..001529f 100644 --- a/src/nsSBCharSetProber.cpp +++ b/src/nsSBCharSetProber.cpp @@ -63,7 +63,7 @@ nsProbingState nsSingleByteCharSetProber::HandleData(const char* aBuf, PRUint32 } if (order < mModel->freqCharCount) { - mFreqChar++; + mFreqChar++; if (mLastOrder < mModel->freqCharCount) { diff --git a/src/nsSBCharSetProber.h b/src/nsSBCharSetProber.h index 42d21b2..0689369 100644 --- a/src/nsSBCharSetProber.h +++ b/src/nsSBCharSetProber.h @@ -190,19 +190,19 @@ extern const SequenceModel Iso_8859_3MalteseModel; extern const SequenceModel Windows_1250CzechModel; extern const SequenceModel Iso_8859_2CzechModel; extern const SequenceModel Ibm852CzechModel; -extern const SequenceModel Mac_CentraleuropeCzechModel; +extern const SequenceModel MaccentraleuropeCzechModel; extern const SequenceModel Windows_1250SlovakModel; extern const SequenceModel Iso_8859_2SlovakModel; extern const SequenceModel Ibm852SlovakModel; -extern const SequenceModel Mac_CentraleuropeSlovakModel; +extern const SequenceModel MaccentraleuropeSlovakModel; extern const SequenceModel Windows_1250PolishModel; extern const SequenceModel Iso_8859_2PolishModel; extern const SequenceModel Iso_8859_13PolishModel; extern const SequenceModel Iso_8859_16PolishModel; extern const SequenceModel Ibm852PolishModel; -extern const SequenceModel Mac_CentraleuropePolishModel; +extern const SequenceModel MaccentraleuropePolishModel; extern const SequenceModel Iso_8859_1FinnishModel; extern const SequenceModel Iso_8859_4FinnishModel; @@ -222,7 +222,7 @@ extern const SequenceModel Iso_8859_2CroatianModel; extern const SequenceModel Iso_8859_13CroatianModel; extern const SequenceModel Iso_8859_16CroatianModel; extern const SequenceModel Ibm852CroatianModel; -extern const SequenceModel Mac_CentraleuropeCroatianModel; +extern const SequenceModel MaccentraleuropeCroatianModel; extern const SequenceModel Windows_1252EstonianModel; extern const SequenceModel Windows_1257EstonianModel; @@ -244,7 +244,7 @@ extern const SequenceModel Windows_1250SloveneModel; extern const SequenceModel Iso_8859_2SloveneModel; extern const SequenceModel Iso_8859_16SloveneModel; extern const SequenceModel Ibm852SloveneModel; -extern const SequenceModel Mac_CentraleuropeSloveneModel; +extern const SequenceModel MaccentraleuropeSloveneModel; extern const SequenceModel Iso_8859_1SwedishModel; extern const SequenceModel Iso_8859_4SwedishModel; diff --git a/src/nsSJISProber.cpp b/src/nsSJISProber.cpp index c7842f6..1c354a7 100644 --- a/src/nsSJISProber.cpp +++ b/src/nsSJISProber.cpp @@ -52,7 +52,7 @@ void nsSJISProber::Reset(void) nsProbingState nsSJISProber::HandleData(const char* aBuf, PRUint32 aLen) { - nsSMState codingState; + PRUint32 codingState; for (PRUint32 i = 0; i < aLen; i++) { diff --git a/src/nsUTF8Prober.cpp b/src/nsUTF8Prober.cpp index ab8d9f7..93677de 100644 --- a/src/nsUTF8Prober.cpp +++ b/src/nsUTF8Prober.cpp @@ -46,7 +46,7 @@ void nsUTF8Prober::Reset(void) nsProbingState nsUTF8Prober::HandleData(const char* aBuf, PRUint32 aLen) { - nsSMState codingState; + PRUint32 codingState; for (PRUint32 i = 0; i < aLen; i++) { diff --git a/src/nsUniversalDetector.cpp b/src/nsUniversalDetector.cpp index 9711618..29159a8 100644 --- a/src/nsUniversalDetector.cpp +++ b/src/nsUniversalDetector.cpp @@ -55,6 +55,7 @@ nsUniversalDetector::nsUniversalDetector(PRUint32 aLanguageFilter) mStart = PR_TRUE; mDetectedCharset = nsnull; + mDetectedConfidence = 0.0; mGotData = PR_FALSE; mInputState = ePureAscii; mLastChar = '\0'; @@ -83,6 +84,7 @@ nsUniversalDetector::Reset() mStart = PR_TRUE; mDetectedCharset = nsnull; + mDetectedConfidence = 0.0; mGotData = PR_FALSE; mInputState = ePureAscii; mLastChar = '\0'; @@ -117,14 +119,18 @@ nsresult nsUniversalDetector::HandleData(const char* aBuf, PRUint32 aLen) switch (aBuf[0]) { case '\xEF': - if (('\xBB' == aBuf[1]) && ('\xBF' == aBuf[2])) + if (('\xBB' == aBuf[1]) && ('\xBF' == aBuf[2])) { /* EF BB BF: UTF-8 encoded BOM. */ - mDetectedCharset = "UTF-8"; + mDetectedCharset = "UTF-8-SIG"; + mDetectedConfidence = 0.99; + } break; case '\xFE': - if ('\xFF' == aBuf[1]) + if ('\xFF' == aBuf[1]) { /* FE FF: UTF-16, big endian BOM. */ mDetectedCharset = "UTF-16"; + mDetectedConfidence = 0.99; + } break; case '\xFF': if ('\xFE' == aBuf[1]) @@ -135,11 +141,13 @@ nsresult nsUniversalDetector::HandleData(const char* aBuf, PRUint32 aLen) { /* FF FE 00 00: UTF-32 (LE). */ mDetectedCharset = "UTF-32"; + mDetectedConfidence = 0.99; } else { /* FF FE: UTF-16, little endian BOM. */ mDetectedCharset = "UTF-16"; + mDetectedConfidence = 0.99; } } break; @@ -151,6 +159,7 @@ nsresult nsUniversalDetector::HandleData(const char* aBuf, PRUint32 aLen) { /* 00 00 FE FF: UTF-32 (BE). */ mDetectedCharset = "UTF-32"; + mDetectedConfidence = 0.99; } break; } @@ -241,16 +250,7 @@ nsresult nsUniversalDetector::HandleData(const char* aBuf, PRUint32 aLen) { mDone = PR_TRUE; mDetectedCharset = mEscCharSetProber->GetCharSetName(); - } - else if (mNbspFound) - { - mDetectedCharset = "ISO-8859-1"; - } - else - { - /* ASCII with the ESC character (or the sequence "~{") is still - * ASCII until proven otherwise. */ - mDetectedCharset = "ASCII"; + mDetectedConfidence = mEscCharSetProber->GetConfidence(); } break; case eHighbyte: @@ -263,6 +263,7 @@ nsresult nsUniversalDetector::HandleData(const char* aBuf, PRUint32 aLen) { mDone = PR_TRUE; mDetectedCharset = mCharSetProbers[i]->GetCharSetName(); + mDetectedConfidence = mCharSetProbers[i]->GetConfidence(); return NS_OK; } } @@ -270,17 +271,6 @@ nsresult nsUniversalDetector::HandleData(const char* aBuf, PRUint32 aLen) break; default: - if (mNbspFound) - { - /* ISO-8859-1 is a good result candidate for ASCII + NBSP. - * (though it could have been any ISO-8859 encoding). */ - mDetectedCharset = "ISO-8859-1"; - } - else - { - /* Pure ASCII */ - mDetectedCharset = "ASCII"; - } break; } return NS_OK; @@ -297,10 +287,35 @@ void nsUniversalDetector::DataEnd() return; } + if (! mDetectedCharset) + { + switch (mInputState) + { + case eEscAscii: + case ePureAscii: + if (mNbspFound) + { + /* ISO-8859-1 is a good result candidate for ASCII + NBSP. + * (though it could have been any ISO-8859 encoding). */ + mDetectedCharset = "ISO-8859-1"; + mDetectedConfidence = 1.0; + } + else + { + /* ASCII with the ESC character (or the sequence "~{") is still + * ASCII until proven otherwise. */ + mDetectedCharset = "ASCII"; + mDetectedConfidence = 1.0; + } + default: + break; + } + } + if (mDetectedCharset) { mDone = PR_TRUE; - Report(mDetectedCharset); + Report(mDetectedCharset, mDetectedConfidence); return; } @@ -326,7 +341,7 @@ void nsUniversalDetector::DataEnd() } //do not report anything because we are not confident of it, that's in fact a negative answer if (maxProberConfidence > MINIMUM_THRESHOLD) - Report(mCharSetProbers[maxProber]->GetCharSetName()); + Report(mCharSetProbers[maxProber]->GetCharSetName(), mCharSetProbers[maxProber]->GetConfidence()); } break; case eEscAscii: diff --git a/src/nsUniversalDetector.h b/src/nsUniversalDetector.h index 9f0a4b1..9ac2db7 100644 --- a/src/nsUniversalDetector.h +++ b/src/nsUniversalDetector.h @@ -69,7 +69,7 @@ class nsUniversalDetector { virtual void DataEnd(void); protected: - virtual void Report(const char* aCharset) = 0; + virtual void Report(const char* aCharset, float aConfidence) = 0; virtual void Reset(); nsInputState mInputState; PRBool mNbspFound; @@ -79,6 +79,7 @@ class nsUniversalDetector { PRBool mGotData; char mLastChar; const char * mDetectedCharset; + float mDetectedConfidence; PRInt32 mBestGuess; PRUint32 mLanguageFilter; diff --git a/src/symbols.cmake b/src/symbols.cmake index c7b1652..b958a35 100644 --- a/src/symbols.cmake +++ b/src/symbols.cmake @@ -6,6 +6,7 @@ set( uchardet_data_end uchardet_reset uchardet_get_charset + uchardet_get_confidence ) set (LINK_FLAGS "") diff --git a/src/tools/CMakeLists.txt b/src/tools/CMakeLists.txt index fa15eb2..ab69f33 100644 --- a/src/tools/CMakeLists.txt +++ b/src/tools/CMakeLists.txt @@ -3,6 +3,19 @@ set( uchardet.cpp ) +if (CMAKE_BUILD_TYPE MATCHES Debug) + add_definitions( + -O0 + -g3 + -fsanitize=address + ) + set(CMAKE_CXX_FLAGS "-fsanitize=address") + set(CMAKE_C_FLAGS "-fsanitize=address") + set(CMAKE_EXE_LINKER_FLAGS "-fsanitize=address") + set(CMAKE_MODULE_LINKER_FLAGS "-fsanitize=address") + set(CMAKE_SHARED_LINKER_FLAGS "-fsanitize=address") +endif (CMAKE_BUILD_TYPE MATCHES Debug) + set(UCHARDET_BINARY uchardet) add_executable( diff --git a/src/tools/uchardet.cpp b/src/tools/uchardet.cpp index f2aabfd..670e84a 100644 --- a/src/tools/uchardet.cpp +++ b/src/tools/uchardet.cpp @@ -57,7 +57,7 @@ void detect(FILE * fp) { size_t len = fread(buffer, 1, BUFFER_SIZE, fp); int retval = uchardet_handle_data(handle, buffer, len); - if (retval != 0) + if (retval == HANDLE_DATA_RESULT_ERROR) { fprintf(stderr, "Handle data error.\n"); exit(1); @@ -66,8 +66,9 @@ void detect(FILE * fp) uchardet_data_end(handle); const char * charset = uchardet_get_charset(handle); + float confidence = uchardet_get_confidence(handle); if (*charset) - printf("%s\n", charset); + printf("{ encoding=%s, confidence=%f }\n", charset, confidence); else printf("unknown\n"); diff --git a/src/uchardet.cpp b/src/uchardet.cpp index f1951d1..5cfc3bf 100644 --- a/src/uchardet.cpp +++ b/src/uchardet.cpp @@ -44,25 +44,29 @@ class HandleUniversalDetector : public nsUniversalDetector { protected: char *m_charset; - + float m_confidence; public: HandleUniversalDetector() : nsUniversalDetector(NS_FILTER_ALL) , m_charset(0) { + m_confidence = 0.0; } virtual ~HandleUniversalDetector() { - if (m_charset) + if (m_charset) { free(m_charset); + m_confidence = 0.0; + } } - virtual void Report(const char* charset) + virtual void Report(const char* charset, float confidence) { if (m_charset) free(m_charset); m_charset = strdup(charset); + m_confidence = confidence; } virtual void Reset() @@ -71,12 +75,21 @@ class HandleUniversalDetector : public nsUniversalDetector if (m_charset) free(m_charset); m_charset = strdup(""); + m_confidence = 0.0; } const char* GetCharset() const { return m_charset? m_charset : ""; } + + float GetConfidence() { + return m_confidence; + } + + PRBool HasDone() { + return mDone; + } }; uchardet_t uchardet_new(void) @@ -92,7 +105,15 @@ void uchardet_delete(uchardet_t ud) int uchardet_handle_data(uchardet_t ud, const char * data, size_t len) { nsresult ret = reinterpret_cast(ud)->HandleData(data, (PRUint32)len); - return (ret != NS_OK); + if (ret == NS_ERROR_OUT_OF_MEMORY) { + return HANDLE_DATA_RESULT_ERROR; + } + + if (reinterpret_cast(ud)->HasDone()) { + return HANDLE_DATA_RESULT_DETECTED; + } + + return HANDLE_DATA_RESULT_NEED_MORE_DATA; } void uchardet_data_end(uchardet_t ud) @@ -109,3 +130,8 @@ const char* uchardet_get_charset(uchardet_t ud) { return reinterpret_cast(ud)->GetCharset(); } + +float uchardet_get_confidence(uchardet_t ud) +{ + return reinterpret_cast(ud)->GetConfidence(); +} diff --git a/src/uchardet.h b/src/uchardet.h index 0ad1de1..5398cc6 100644 --- a/src/uchardet.h +++ b/src/uchardet.h @@ -61,13 +61,17 @@ uchardet_t uchardet_new(void); */ void uchardet_delete(uchardet_t ud); +#define HANDLE_DATA_RESULT_ERROR -1 +#define HANDLE_DATA_RESULT_DETECTED 0 +#define HANDLE_DATA_RESULT_NEED_MORE_DATA 1 + /** * Feed data to an encoding detector. * The detector is able to shortcut processing when it reaches certainty * for an encoding, so you should not worry about limiting input data. * As far as you should be concerned: the more the better. * - * @param ud [in] handle of a instance of uchardet + * @param ud [in] handle of an instance of uchardet * @param data [in] data * @param len [in] number of byte of data * @return non-zero number on failure. @@ -75,24 +79,26 @@ void uchardet_delete(uchardet_t ud); int uchardet_handle_data(uchardet_t ud, const char * data, size_t len); /** - * Notify an end of data to an encoding detctor. - * @param ud [in] handle of a instance of uchardet + * Notify an end of data to an encoding detector. + * @param ud [in] handle of an instance of uchardet */ void uchardet_data_end(uchardet_t ud); /** * Reset an encoding detector. - * @param ud [in] handle of a instance of uchardet + * @param ud [in] handle of an instance of uchardet */ void uchardet_reset(uchardet_t ud); /** * Get an iconv-compatible name of the encoding that was detected. - * @param ud [in] handle of a instance of uchardet + * @param ud [in] handle of an instance of uchardet * @return name of charset on success and "" on failure. */ const char * uchardet_get_charset(uchardet_t ud); +float uchardet_get_confidence(uchardet_t ud); + #ifdef __cplusplus } #endif diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index a059044..5b27e2d 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -3,6 +3,19 @@ set( uchardet-tests.c ) +if (CMAKE_BUILD_TYPE MATCHES Debug) + add_definitions( + -O0 + -g3 + -fsanitize=address + ) + set(CMAKE_CXX_FLAGS "-fsanitize=address") + set(CMAKE_C_FLAGS "-fsanitize=address") + set(CMAKE_EXE_LINKER_FLAGS "-fsanitize=address") + set(CMAKE_MODULE_LINKER_FLAGS "-fsanitize=address") + set(CMAKE_SHARED_LINKER_FLAGS "-fsanitize=address") +endif (CMAKE_BUILD_TYPE MATCHES Debug) + add_executable( uchardet-tests ${UCHARDET_TEST_SOURCES} diff --git a/test/cs/mac-centraleurope.txt b/test/cs/maccentraleurope.txt similarity index 100% rename from test/cs/mac-centraleurope.txt rename to test/cs/maccentraleurope.txt diff --git a/test/hr/mac-centraleurope.txt b/test/hr/maccentraleurope.txt similarity index 100% rename from test/hr/mac-centraleurope.txt rename to test/hr/maccentraleurope.txt diff --git a/test/pl/mac-centraleurope.txt b/test/pl/maccentraleurope.txt similarity index 100% rename from test/pl/mac-centraleurope.txt rename to test/pl/maccentraleurope.txt diff --git a/test/ru/mac-cyrillic.txt b/test/ru/maccyrillic.txt similarity index 100% rename from test/ru/mac-cyrillic.txt rename to test/ru/maccyrillic.txt diff --git a/test/sk/mac-centraleurope.txt b/test/sk/maccentraleurope.txt similarity index 100% rename from test/sk/mac-centraleurope.txt rename to test/sk/maccentraleurope.txt diff --git a/test/sl/mac-centraleurope.txt b/test/sl/maccentraleurope.txt similarity index 100% rename from test/sl/mac-centraleurope.txt rename to test/sl/maccentraleurope.txt diff --git a/test/uchardet-tests.c b/test/uchardet-tests.c index 180ac15..c7fa0a0 100644 --- a/test/uchardet-tests.c +++ b/test/uchardet-tests.c @@ -56,7 +56,7 @@ detect(FILE *fp) { size_t len = fread(buffer, 1, BUFFER_SIZE, fp); int retval = uchardet_handle_data(handle, buffer, len); - if (retval != 0) + if (retval == HANDLE_DATA_RESULT_ERROR) { fprintf(stderr, "uchardet-tests: handle data error.\n"); @@ -120,6 +120,10 @@ main(int argc, char ** argv) /* In a unit test, 0 means success, other returned values mean failure. */ success = (strcmp(charset, expected_charset) != 0); + + // if (success != 0) { + // printf("expected_charset=%s, got=%s\n", expected_charset, charset); + // } free(charset); free(filename);