build(deps): bump requests from 2.20.0 to 2.31.0 in /script

.gitignore

@@ -1 +1,4 @@
 __pycache__/
+/build/
+/.vscode/
+.DS_Store

CMakeLists.txt

@@ -1,5 +1,6 @@
 ######## Project settings
 cmake_minimum_required(VERSION 2.8.5)
+include(CheckCCompilerFlag)
 set (PACKAGE_NAME uchardet)
 project (${PACKAGE_NAME} CXX C)
 enable_testing()
@@ -19,27 +20,40 @@ set (
 	UCHARDET_VERSION
 	${UCHARDET_VERSION_MAJOR}.${UCHARDET_VERSION_MINOR}.${UCHARDET_VERSION_REVISION}${version_suffix}
 )
-
-######## Windows
-
-#if (WIN32)
-#	set(CMAKE_SHARED_LIBRARY_PREFIX ${CMAKE_INSTALL_PREFIX})
-#	set(CMAKE_STATIC_LIBRARY_PREFIX ${CMAKE_INSTALL_PREFIX})
-#endif (WIN32)
+set(CMAKE_CXX_STANDARD 11)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
 ######## Directory
 
 include(GNUInstallDirs)
 
 ######## Configuration
 
-option(BUILD_BINARY "Build executable" ON)
-option(BUILD_SHARED_LIBS "Build shared library and link executable to it" ON)
+option(BUILD_BINARY "Build the CLI tool." ON)
+option(BUILD_SHARED_LIBS "Build shared library and link executable to it." ON)
+option(CHECK_SSE2 "Check and enable SSE2 extensions if supported. Disabling SSE on platforms which support it may decrease performances." ON)
+set(TARGET_ARCHITECTURE "" CACHE STRING "Target CPU architecture. It is autodetected if not specified.")
 
 if (BUILD_SHARED_LIBS)
 	option(BUILD_STATIC "Build static library" ON)
 endif (BUILD_SHARED_LIBS)
 
+if (TARGET_ARCHITECTURE STREQUAL "")
+    string(TOLOWER ${CMAKE_SYSTEM_PROCESSOR} TARGET_ARCHITECTURE)
+endif (TARGET_ARCHITECTURE STREQUAL "")
+
+if (TARGET_ARCHITECTURE MATCHES ".*(x86)|(amd).*")
+    CHECK_C_COMPILER_FLAG(-msse2 SUPPORTS_CFLAG_SSE2)
+    CHECK_C_COMPILER_FLAG(-mfpmath=sse SUPPORTS_CFLAG_SSE_MATH)
+    if (CHECK_SSE2 AND SUPPORTS_CFLAG_SSE2 AND SUPPORTS_CFLAG_SSE_MATH)
+        set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -msse2 -mfpmath=sse")
+        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse2 -mfpmath=sse")
+    else (CHECK_SSE2 AND SUPPORTS_CFLAG_SSE2 AND SUPPORTS_CFLAG_SSE_MATH)
+        set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -ffloat-store")
+        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ffloat-store")
+    endif (CHECK_SSE2 AND SUPPORTS_CFLAG_SSE2 AND SUPPORTS_CFLAG_SSE_MATH)
+endif (TARGET_ARCHITECTURE MATCHES ".*(x86)|(amd).*")
+
 configure_file(
 	uchardet.pc.in
 	uchardet.pc

INSTALL

@@ -1,4 +1,4 @@
-# INSTALL uchardet
+# Building uchardet (generic)
 
 `uchardet` uses a typical cmake installation.
 
@@ -14,3 +14,13 @@ Alternatively, use `ccmake`, curses interface to `cmake`.
 * Install with `make install`.
 
 Read `README` for more details on uchardet.
+
+# Building uchardet on Windows
+
+The above procedure is generic, which means it should work on any platform.
+In particular, it works well on Linux.
+
+The procedure is the same on Windows, but if you want more details (for
+instance which tools to use in order to run CMake on Windows, compiler
+information, etc.), the following link may be useful:
+https://github.com/BYVoid/uchardet/issues/39#issuecomment-353873891

README.md

@@ -1,5 +1,7 @@
 # uchardet
 
+Forked from [freedesktop/uchardet](https://github.com/freedesktop/uchardet)
+
 [uchardet](https://www.freedesktop.org/wiki/Software/uchardet/) is an encoding detector library, which takes a sequence of bytes in an unknown character encoding without any additional information, and attempts to determine the encoding of the text. Returned encoding names are [iconv](https://www.gnu.org/software/libiconv/)-compatible.
 
 uchardet started as a C language binding of the original C++ implementation of the universal charset detection library by Mozilla. It can now detect more charsets, and more reliably than the original implementation.
@@ -32,12 +34,12 @@ Techniques used by universalchardet are described at http://www.mozilla.org/proj
     * ISO-8859-16
     * Windows-1250
     * IBM852
-    * MAC-CENTRALEUROPE
+    * MacCentralEurope
   * Czech
     * Windows-1250
     * ISO-8859-2
     * IBM852
-    * MAC-CENTRALEUROPE
+    * MacCentralEurope
   * Danish
     * ISO-8859-1
     * ISO-8859-15
@@ -109,7 +111,7 @@ Techniques used by universalchardet are described at http://www.mozilla.org/proj
     * ISO-8859-16
     * Windows-1250
     * IBM852
-    * MAC-CENTRALEUROPE
+    * MacCentralEurope
   * Portuguese
     * ISO-8859-1
     * ISO-8859-9
@@ -131,13 +133,13 @@ Techniques used by universalchardet are described at http://www.mozilla.org/proj
     * Windows-1250
     * ISO-8859-2
     * IBM852
-    * MAC-CENTRALEUROPE
+    * MacCentralEurope
   * Slovene
     * ISO-8859-2
     * ISO-8859-16
     * Windows-1250
     * IBM852
-    * MAC-CENTRALEUROPE
+    * MacCentralEurope
   * Spanish
     * ISO-8859-1
     * ISO-8859-15
@@ -162,57 +164,19 @@ Techniques used by universalchardet are described at http://www.mozilla.org/proj
 
 ## Installation
 
-### Debian/Ubuntu/Mint
-
-    apt-get install uchardet libuchardet-dev
-
-### Mageia
-
-    urpmi libuchardet libuchardet-devel
-
-### Fedora
-
-    dnf install uchardet uchardet-devel
-
-### Mac
-
-    brew install uchardet
-
 ### Build from source
 
-Releases are available from:
-https://www.freedesktop.org/software/uchardet/releases/
-
 If you prefer a development version, clone the git repository:
 
-    git clone git://anongit.freedesktop.org/uchardet/uchardet
+    git clone https://github.com/PyYoshi/uchardet.git
 
-The source can be browsed at: https://cgit.freedesktop.org/uchardet/uchardet/
+The source can be browsed at: https://github.com/PyYoshi/uchardet
 
-    cmake .
+    mkdir build/ && cd build/
+    cmake ..
     make
     make install
 
-### Build with flatpak-builder
-
-Here is a working "module" section to include in your Flatpak's json manifest:
-
-```
-"modules": [
-    {
-        "name": "uchardet",
-        "buildsystem": "cmake",
-        "builddir": true,
-        "config-opts": [ "-DCMAKE_INSTALL_LIBDIR=lib" ],
-        "sources": [
-            {
-                ...
-            }
-        ]
-    }
-]
-```
-
 ## Usage
 
 ### Command Line
@@ -233,19 +197,7 @@ Options:
  ```
 ### Library
 
-See [uchardet.h](https://cgit.freedesktop.org/uchardet/uchardet/tree/src/uchardet.h)
-
-## Related Projects
-
-  * [python-chardet](https://github.com/chardet/chardet) Python port
-  * [ruby-rchardet](http://rubyforge.org/projects/chardet/) Ruby port
-  * [juniversalchardet](http://code.google.com/p/juniversalchardet/) Java port of universalchardet
-  * [jchardet](http://jchardet.sourceforge.net/) Java port of chardet
-  * [nuniversalchardet](http://code.google.com/p/nuniversalchardet/) C# port of universalchardet
-  * [nchardet](http://www.conceptdevelopment.net/Localization/NCharDet/) C# port of chardet
-  * [uchardet-enhanced](https://bitbucket.org/medoc/uchardet-enhanced) A fork of mozilla universalchardet
-  * [rust-uchardet](https://github.com/emk/rust-uchardet) Rust language binding of uchardet
-  * [libchardet](https://ftp.oops.org/pub/oops/libchardet/) Another C/C++ API wrapping Mozilla code.
+See [uchardet.h](https://github.com/PyYoshi/uchardet/blob/cchardet/src/uchardet.h)
 
 ## Licenses
 

script/BuildLangModelLogs/LangEstonianModel.log

@@ -156,4 +156,4 @@ First 512 (typical positive ratio): 0.9972721312183132
 Next 512 (512-1024): 9.687262863877811e-05
 Rest: -5.204170427930421e-18
 
-- Processing end: 2016-09-26 23:47:54.561846
+- Processing end: 2016-09-26 23:47:54.561846

script/charsets/mac-centraleurope.py

@@ -40,7 +40,7 @@
 
 from codepoints import *
 
-name = 'MAC-CENTRALEUROPE'
+name = 'MacCentralEurope'
 aliases = []
 
 language = \

script/gen.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+python ./BuildLangModel.py ar --max-page=100 --max-depth=4
+python ./BuildLangModel.py cs --max-page=100 --max-depth=4
+python ./BuildLangModel.py da --max-page=100 --max-depth=4
+python ./BuildLangModel.py de --max-page=100 --max-depth=4
+python ./BuildLangModel.py el --max-page=100 --max-depth=4
+python ./BuildLangModel.py eo --max-page=100 --max-depth=4
+python ./BuildLangModel.py es --max-page=100 --max-depth=4
+python ./BuildLangModel.py et --max-page=100 --max-depth=4
+python ./BuildLangModel.py fi --max-page=100 --max-depth=4
+python ./BuildLangModel.py fr --max-page=100 --max-depth=4
+python ./BuildLangModel.py ga --max-page=100 --max-depth=4
+python ./BuildLangModel.py hr --max-page=100 --max-depth=4
+python ./BuildLangModel.py hu --max-page=100 --max-depth=4
+python ./BuildLangModel.py it --max-page=100 --max-depth=4
+python ./BuildLangModel.py lt --max-page=100 --max-depth=4
+python ./BuildLangModel.py lv --max-page=100 --max-depth=4
+python ./BuildLangModel.py mt --max-page=100 --max-depth=4
+python ./BuildLangModel.py pl --max-page=100 --max-depth=4
+python ./BuildLangModel.py pt --max-page=100 --max-depth=4
+python ./BuildLangModel.py ro --max-page=100 --max-depth=4
+python ./BuildLangModel.py sk --max-page=100 --max-depth=4
+python ./BuildLangModel.py sl --max-page=100 --max-depth=4
+python ./BuildLangModel.py sv --max-page=100 --max-depth=4
+python ./BuildLangModel.py th --max-page=100 --max-depth=4
+python ./BuildLangModel.py tr --max-page=100 --max-depth=4
+python ./BuildLangModel.py vi --max-page=100 --max-depth=4

script/langs/cs.py

@@ -50,7 +50,7 @@
 # ASCII characters are also used in French.
 use_ascii = True
 # The charsets we want to support and create data for.
-charsets = ['ISO-8859-2', 'Windows-1250', 'IBM852', 'MAC-CENTRALEUROPE']
+charsets = ['ISO-8859-2', 'Windows-1250', 'IBM852', 'MacCentralEurope']
 
 ## Optional Properties ##
 

script/langs/hr.py

@@ -46,7 +46,7 @@
 code = 'hr'
 use_ascii = True
 charsets = ['ISO-8859-2', 'ISO-8859-13', 'ISO-8859-16',
-            'Windows-1250', 'IBM852', 'MAC-CENTRALEUROPE']
+            'Windows-1250', 'IBM852', 'MacCentralEurope']
 
 ## Optional Properties ##
 

script/langs/pl.py

@@ -51,7 +51,7 @@
 use_ascii = True
 # The charsets we want to support and create data for.
 charsets = ['ISO-8859-2', 'ISO-8859-13', 'ISO-8859-16',
-            'Windows-1250', 'IBM852', 'MAC-CENTRALEUROPE']
+            'Windows-1250', 'IBM852', 'MacCentralEurope']
 
 ## Optional Properties ##
 

script/langs/sk.py

@@ -50,7 +50,7 @@
 # ASCII characters are also used in French.
 use_ascii = True
 # The charsets we want to support and create data for.
-charsets = ['ISO-8859-2', 'Windows-1250', 'IBM852', 'MAC-CENTRALEUROPE']
+charsets = ['ISO-8859-2', 'Windows-1250', 'IBM852', 'MacCentralEurope']
 
 ## Optional Properties ##
 

script/langs/sl.py

@@ -46,7 +46,7 @@
 code = 'sl'
 use_ascii = True
 charsets = ['ISO-8859-2', 'ISO-8859-16',
-            'Windows-1250', 'IBM852', 'MAC-CENTRALEUROPE']
+            'Windows-1250', 'IBM852', 'MacCentralEurope']
 
 ## Optional Properties ##
 

script/requirements.txt

@@ -0,0 +1,2 @@
+requests==2.31.0
+wikipedia==1.4.0

src/CMakeLists.txt

@@ -72,7 +72,13 @@ if (CMAKE_BUILD_TYPE MATCHES Debug)
 	add_definitions(
 		-O0
 		-g3
+		-fsanitize=address
 	)
+	set(CMAKE_CXX_FLAGS "-fsanitize=address")
+	set(CMAKE_C_FLAGS "-fsanitize=address")
+	set(CMAKE_EXE_LINKER_FLAGS "-fsanitize=address")
+	set(CMAKE_MODULE_LINKER_FLAGS "-fsanitize=address")
+	set(CMAKE_SHARED_LINKER_FLAGS "-fsanitize=address")
 endif (CMAKE_BUILD_TYPE MATCHES Debug)
 
 add_library(
@@ -152,4 +158,9 @@ include(symbols.cmake)
 
 if (BUILD_BINARY)
 	add_subdirectory(tools)
+	add_subdirectory(fuzzing)
 endif (BUILD_BINARY)
+
+if (BUILD_FUZZER)
+	add_subdirectory(fuzzing)
+endif (BUILD_FUZZER)

src/EUCTWFreq.tab

@@ -57,7 +57,7 @@
 #define EUCTW_TYPICAL_DISTRIBUTION_RATIO (float)0.75
 
 //Char to FreqOrder table , 
-#define EUCTW_TABLE_SIZE  8102
+#define EUCTW_TABLE_SIZE  5376
 
 static const PRInt16 EUCTWCharToFreqOrder[] =
 {

src/LangModels/LangCroatianModel.cpp

@@ -124,7 +124,7 @@ static const unsigned char Iso_8859_16_CharToOrderMap[] =
 };
 /*X0  X1  X2  X3  X4  X5  X6  X7  X8  X9  XA  XB  XC  XD  XE  XF */
 
-static const unsigned char Mac_Centraleurope_CharToOrderMap[] =
+static const unsigned char Maccentraleurope_CharToOrderMap[] =
 {
   CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,RET,CTR,CTR,RET,CTR,CTR, /* 0X */
   CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR, /* 1X */
@@ -261,14 +261,14 @@ const SequenceModel Iso_8859_16CroatianModel =
   "ISO-8859-16"
 };
 
-const SequenceModel Mac_CentraleuropeCroatianModel =
+const SequenceModel MaccentraleuropeCroatianModel =
 {
-  Mac_Centraleurope_CharToOrderMap,
+  Maccentraleurope_CharToOrderMap,
   CroatianLangModel,
   31,
   (float)0.9989731099787131,
   PR_TRUE,
-  "MAC-CENTRALEUROPE"
+  "MacCentralEurope"
 };
 
 const SequenceModel Iso_8859_13CroatianModel =

src/LangModels/LangCzechModel.cpp

@@ -82,7 +82,7 @@ static const unsigned char Windows_1250_CharToOrderMap[] =
 };
 /*X0  X1  X2  X3  X4  X5  X6  X7  X8  X9  XA  XB  XC  XD  XE  XF */
 
-static const unsigned char Mac_Centraleurope_CharToOrderMap[] =
+static const unsigned char Maccentraleurope_CharToOrderMap[] =
 {
   CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,RET,CTR,CTR,RET,CTR,CTR, /* 0X */
   CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR, /* 1X */
@@ -250,14 +250,14 @@ const SequenceModel Windows_1250CzechModel =
   "WINDOWS-1250"
 };
 
-const SequenceModel Mac_CentraleuropeCzechModel =
+const SequenceModel MaccentraleuropeCzechModel =
 {
-  Mac_Centraleurope_CharToOrderMap,
+  Maccentraleurope_CharToOrderMap,
   CzechLangModel,
   41,
   (float)0.9786035192432675,
   PR_TRUE,
-  "MAC-CENTRALEUROPE"
+  "MacCentralEurope"
 };
 
 const SequenceModel Ibm852CzechModel =
@@ -278,4 +278,4 @@ const SequenceModel Iso_8859_2CzechModel =
   (float)0.9786035192432675,
   PR_TRUE,
   "ISO-8859-2"
-};
+};