Rooba
/
RapidFuzz
mirror of https://github.com/maxbachmann/RapidFuzz.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

drop Python2.7 support

This commit is contained in:
Max Bachmann 2021-03-08 19:44:28 +01:00
parent e124f4f32e
commit bfd968b606
7 changed files with 10 additions and 608 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

122
.github/workflows/pythonbuild.yml vendored
View File

@ -8,47 +8,7 @@ on:
- published
jobs:
test_python:
name: linting and tests on Python ${{ matrix.python-version }}
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [ubuntu-18.04, macos-latest]
python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9]
steps:
- uses: actions/checkout@v2
with:
submodules: 'true'
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v1
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install wheel
- name: Lint with flake8
run: |
pip install flake8
# stop the build if there are Python syntax errors or undefined names
flake8 src --count --select=E9,F63,F7,F82 --show-source --statistics
# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
flake8 src --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
- name: Run Unit Tests
run: |
pip install .
pip install pytest hypothesis pandas
pytest
build_wheels:
needs: [test_python]
name: Build wheel on ${{ matrix.os }}
runs-on: ${{ matrix.os }}
strategy:
@ -56,7 +16,9 @@ jobs:
matrix:
os: [ubuntu-18.04, windows-latest, macos-latest]
env:
CIBW_SKIP: cp27-win* pp27-win*
CIBW_SKIP: cp27-* pp27-*
CIBW_TEST_REQUIRES: pytest hypothesis pandas
CIBW_TEST_COMMAND: pytest {package}/tests
steps:
- uses: actions/checkout@v1
@ -70,7 +32,7 @@ jobs:
- name: Install cibuildwheel
run: |
python -m pip install git+https://github.com/joerick/cibuildwheel.git@f6eaa9f
python -m pip install cibuildwheel~=1.10.0
- name: Build wheels
run: |
python -m cibuildwheel --output-dir wheelhouse
@ -79,7 +41,6 @@ jobs:
path: ./wheelhouse/*.whl
build_wheels_manylinux1:
needs: [test_python]
name: Build wheel with manylinux1
runs-on: ubuntu-18.04
strategy:
@ -88,6 +49,8 @@ jobs:
CIBW_BUILD: cp*
CIBW_MANYLINUX_X86_64_IMAGE: manylinux1
CIBW_MANYLINUX_I686_IMAGE: manylinux1
CIBW_TEST_REQUIRES: pytest hypothesis pandas
CIBW_TEST_COMMAND: pytest {package}/tests
steps:
- uses: actions/checkout@v1
@ -101,7 +64,7 @@ jobs:
- name: Install cibuildwheel
run: |
python -m pip install git+https://github.com/joerick/cibuildwheel.git@f6eaa9f
python -m pip install cibuildwheel~=1.10.0
- name: Build wheels
run: |
python -m cibuildwheel --output-dir wheelhouse
@ -109,76 +72,7 @@ jobs:
with:
path: ./wheelhouse/*.whl
build_wheels_python27_windows_64:
needs: [test_python]
name: Build wheel for python2.7 on windows-latest 64 bit
runs-on: windows-latest
env:
CIBW_BUILD: cp27-win_amd64
DISTUTILS_USE_SDK: 1
MSSdk: 1
steps:
- uses: actions/checkout@v1
with:
submodules: 'true'
- uses: actions/setup-python@v1
name: Install Python
with:
python-version: '3.7'
- name: Install cibuildwheel
run: |
python -m pip install git+https://github.com/joerick/cibuildwheel.git@f6eaa9f
- uses: ilammy/msvc-dev-cmd@v1
- name: Build 64-bit wheel
run: python -m cibuildwheel --output-dir wheelhouse
- uses: actions/upload-artifact@v2
with:
path: ./wheelhouse/*.whl
build_wheels_python27_windows_32:
needs: [test_python]
name: Build wheel for python2.7 on windows-latest 32 bit
runs-on: windows-latest
env:
CIBW_BUILD: cp27-win32 pp27-win32
DISTUTILS_USE_SDK: 1
MSSdk: 1
steps:
- uses: actions/checkout@v1
with:
submodules: 'true'
- uses: actions/setup-python@v1
name: Install Python
with:
python-version: '3.7'
- name: Install cibuildwheel
run: |
python -m pip install git+https://github.com/joerick/cibuildwheel.git@f6eaa9f
- uses: ilammy/msvc-dev-cmd@v1
with:
arch: x86
- name: Build 64-bit wheel
run: python -m cibuildwheel --output-dir wheelhouse
- uses: actions/upload-artifact@v2
with:
path: ./wheelhouse/*.whl
build_sdist:
needs: [test_python]
name: Build source distribution
runs-on: ubuntu-latest
steps:
@ -205,7 +99,7 @@ jobs:
deploy-wheels:
if: github.event_name == 'release' && github.event.action == 'published'
needs: [build_wheels, build_wheels_manylinux1, build_wheels_python27_windows_64, build_wheels_python27_windows_32, build_sdist]
needs: [build_wheels, build_wheels_manylinux1, build_sdist]
name: deploy wheels to pypi
runs-on: ubuntu-18.04

2
README.md
View File

@ -52,7 +52,7 @@ RapidFuzz is a fast string matching library for Python and C++, which is using t
## Requirements
- Python 2.7 or later
- Python 3.5 or later
- On Windows the [Visual C++ 2019 redistributable](https://support.microsoft.com/en-us/help/2977003/the-latest-supported-visual-c-downloads) is required
## Installation

4
setup.cfg
View File

@ -11,8 +11,6 @@ long_description_content_type = text/markdown
license = MIT
license_file = LICENSE
classifiers =
Programming Language :: Python :: 2
Programming Language :: Python :: 2.7
Programming Language :: Python :: 3
Programming Language :: Python :: 3.5
Programming Language :: Python :: 3.6
@ -27,7 +25,7 @@ include_package_data = True
package_dir=
=src
packages = find:
python_requires = >=2.7
python_requires = >=3.5
[options.packages.find]
where=src

116
src/cpp_fuzz.hpp
View File

@ -21,13 +21,6 @@ private:
char const* m_error;
};
#if PY_VERSION_HEX < PYTHON_VERSION(3, 0, 0)
enum StringKind{
CHAR_STRING,
UNICODE_STRING
};
#endif
struct proc_string {
int kind;
void* data;
@ -38,7 +31,6 @@ static proc_string convert_string(PyObject* py_str)
{
proc_string str = {0, NULL, 0};
#if PY_VERSION_HEX >= PYTHON_VERSION(3, 0, 0)
if (!PyUnicode_Check(py_str)) {
throw PythonTypeError("choice must be a String or None");
}
@ -54,26 +46,10 @@ static proc_string convert_string(PyObject* py_str)
str.kind = PyUnicode_KIND(py_str);
str.data = PyUnicode_DATA(py_str);
str.length = PyUnicode_GET_LENGTH(py_str);
#else /* Python 2 */
if (PyObject_TypeCheck(py_str, &PyString_Type)) {
str.kind = CHAR_STRING;
str.length = PyString_GET_SIZE(py_str);
str.data = (void*)PyString_AS_STRING(py_str);
}
else if (PyObject_TypeCheck(py_str, &PyUnicode_Type)) {
str.kind = UNICODE_STRING;
str.length = PyUnicode_GET_SIZE(py_str);
str.data = (void*)PyUnicode_AS_UNICODE(py_str);
}
else {
throw PythonTypeError("choice must be a String, Unicode or None");
}
#endif
return str;
}
#if PY_VERSION_HEX >= PYTHON_VERSION(3, 0, 0)
#define RATIO_SINGLE(RATIO, RATIO_FUNC) \
template<typename CharT> \
inline double RATIO##_single(proc_string s1, proc_string s2, double score_cutoff) \
@ -99,30 +75,7 @@ inline double RATIO##_single(proc_string s1, proc_string s2, double score_cutoff
); \
} \
}
#else
#define RATIO_SINGLE(RATIO, RATIO_FUNC) \
template<typename CharT> \
inline double RATIO##_single(proc_string s1, proc_string s2, double score_cutoff) \
{ \
switch(s2.kind){ \
case CHAR_STRING: \
return RATIO_FUNC( \
rapidfuzz::basic_string_view<CharT>((CharT*)s1.data, s1.length), \
rapidfuzz::basic_string_view<uint8_t>((uint8_t*)s2.data, s2.length), \
score_cutoff \
); \
default: \
return RATIO_FUNC( \
rapidfuzz::basic_string_view<CharT>((CharT*)s1.data, s1.length), \
rapidfuzz::basic_string_view<Py_UNICODE>((Py_UNICODE*)s2.data, s2.length), \
score_cutoff \
); \
} \
}
#endif
#if PY_VERSION_HEX >= PYTHON_VERSION(3, 0, 0)
#define RATIO_IMPL(RATIO, RATIO_FUNC) \
double RATIO##_impl(PyObject* s1, PyObject* s2, double score_cutoff) \
{ \
@ -141,27 +94,7 @@ double RATIO##_impl(PyObject* s1, PyObject* s2, double score_cutoff) \
return RATIO##_single<uint32_t>(c_s1, c_s2, score_cutoff); \
} \
}
#else
#define RATIO_IMPL(RATIO, RATIO_FUNC) \
double RATIO##_impl(PyObject* s1, PyObject* s2, double score_cutoff) \
{ \
proc_string c_s1 = convert_string(s1); \
if (c_s1.data == NULL) return 0.0; \
\
proc_string c_s2 = convert_string(s2); \
if (c_s2.data == NULL) return 0.0; \
\
switch(c_s1.kind){ \
case CHAR_STRING: \
return RATIO##_single<uint8_t>(c_s1, c_s2, score_cutoff); \
default: \
return RATIO##_single<Py_UNICODE>(c_s1, c_s2, score_cutoff); \
} \
}
#endif
#if PY_VERSION_HEX >= PYTHON_VERSION(3, 0, 0)
#define RATIO_SINGLE_DEFAULT_PROCESS(RATIO, RATIO_FUNC) \
template<typename CharT> \
inline double RATIO##_single_default_process( \
@ -200,39 +133,7 @@ inline double RATIO##_single_default_process(
); \
} \
}
#else
#define RATIO_SINGLE_DEFAULT_PROCESS(RATIO, RATIO_FUNC) \
template<typename CharT> \
inline double RATIO##_single_default_process( \
proc_string s1, proc_string s2, double score_cutoff) \
{ \
switch(s2.kind){ \
case CHAR_STRING: \
return RATIO_FUNC( \
utils::default_process( \
rapidfuzz::basic_string_view<CharT>((CharT*)s1.data, s1.length) \
), \
utils::default_process( \
rapidfuzz::basic_string_view<uint8_t>((uint8_t*)s2.data, s2.length) \
), \
score_cutoff \
); \
default: \
return RATIO_FUNC( \
utils::default_process( \
rapidfuzz::basic_string_view<CharT>((CharT*)s1.data, s1.length) \
), \
utils::default_process( \
rapidfuzz::basic_string_view<Py_UNICODE>((Py_UNICODE*)s2.data, s2.length) \
), \
score_cutoff \
); \
} \
}
#endif
#if PY_VERSION_HEX >= PYTHON_VERSION(3, 0, 0)
#define RATIO_IMPL_DEFAULT_PROCESS(RATIO, RATIO_FUNC) \
double RATIO##_impl_default_process(PyObject* s1, PyObject* s2, double score_cutoff) { \
proc_string c_s1 = convert_string(s1); \
@ -250,23 +151,6 @@ double RATIO##_impl_default_process(PyObject* s1, PyObject* s2, double score_cut
return RATIO##_single_default_process<uint32_t>(c_s1, c_s2, score_cutoff); \
} \
}
#else
#define RATIO_IMPL_DEFAULT_PROCESS(RATIO, RATIO_FUNC) \
double RATIO##_impl_default_process(PyObject* s1, PyObject* s2, double score_cutoff) { \
proc_string c_s1 = convert_string(s1); \
if (c_s1.data == NULL) return 0.0; \
\
proc_string c_s2 = convert_string(s2); \
if (c_s2.data == NULL) return 0.0; \
\
switch(c_s1.kind){ \
case CHAR_STRING: \
return RATIO##_single_default_process<uint8_t>(c_s1, c_s2, score_cutoff); \
default: \
return RATIO##_single_default_process<Py_UNICODE>(c_s1, c_s2, score_cutoff); \
} \
}
#endif
#define RATIO_DEF(RATIO, RATIO_FUNC) \
RATIO_SINGLE_DEFAULT_PROCESS(RATIO, RATIO_FUNC) \

58
src/cpp_process.hpp
View File

@ -53,13 +53,6 @@ struct ExtractComp
}
};
#if PY_VERSION_HEX < PYTHON_VERSION(3, 0, 0)
enum StringKind{
CHAR_STRING,
UNICODE_STRING
};
#endif
struct proc_string {
int kind;
void* data;
@ -70,7 +63,6 @@ static proc_string convert_string(PyObject* py_str)
{
proc_string str = {0, NULL, 0};
#if PY_VERSION_HEX >= PYTHON_VERSION(3, 0, 0)
if (!PyUnicode_Check(py_str)) {
throw PythonTypeError("choice must be a String or None");
}
@ -86,21 +78,6 @@ static proc_string convert_string(PyObject* py_str)
str.kind = PyUnicode_KIND(py_str);
str.data = PyUnicode_DATA(py_str);
str.length = PyUnicode_GET_LENGTH(py_str);
#else /* Python 2 */
if (PyObject_TypeCheck(py_str, &PyString_Type)) {
str.kind = CHAR_STRING;
str.length = PyString_GET_SIZE(py_str);
str.data = (void*)PyString_AS_STRING(py_str);
}
else if (PyObject_TypeCheck(py_str, &PyUnicode_Type)) {
str.kind = UNICODE_STRING;
str.length = PyUnicode_GET_SIZE(py_str);
str.data = (void*)PyUnicode_AS_UNICODE(py_str);
}
else {
throw PythonTypeError("choice must be a String, Unicode or None");
}
#endif
return str;
}
@ -130,7 +107,6 @@ static inline double cached_func_default_process(
CachedScorer* ratio = (CachedScorer*)context;
switch(str.kind){
#if PY_VERSION_HEX >= PYTHON_VERSION(3, 0, 0)
case PyUnicode_1BYTE_KIND:
return ratio->ratio(
utils::default_process(
@ -149,20 +125,6 @@ static inline double cached_func_default_process(
rapidfuzz::basic_string_view<uint32_t>((uint32_t*)str.data, str.length)),
score_cutoff
);
#else
case CHAR_STRING:
return ratio->ratio(
utils::default_process(
rapidfuzz::basic_string_view<uint8_t>((uint8_t*)str.data, str.length)),
score_cutoff
);
case UNICODE_STRING:
return ratio->ratio(
utils::default_process(
rapidfuzz::basic_string_view<Py_UNICODE>((Py_UNICODE*)str.data, str.length)),
score_cutoff
);
#endif
default:
throw std::logic_error("Reached end of control flow in cached_func_default_process");
}
@ -175,7 +137,6 @@ static inline double cached_func(void* context, PyObject* py_str, double score_c
CachedScorer* ratio = (CachedScorer*)context;
switch(str.kind){
#if PY_VERSION_HEX >= PYTHON_VERSION(3, 0, 0)
case PyUnicode_1BYTE_KIND:
return ratio->ratio(
rapidfuzz::basic_string_view<uint8_t>((uint8_t*)str.data, str.length),
@ -191,18 +152,6 @@ static inline double cached_func(void* context, PyObject* py_str, double score_c
rapidfuzz::basic_string_view<uint32_t>((uint32_t*)str.data, str.length),
score_cutoff
);
#else
case CHAR_STRING:
return ratio->ratio(
rapidfuzz::basic_string_view<uint8_t>((uint8_t*)str.data, str.length),
score_cutoff
);
case UNICODE_STRING:
return ratio->ratio(
rapidfuzz::basic_string_view<Py_UNICODE>((Py_UNICODE*)str.data, str.length),
score_cutoff
);
#endif
default:
throw std::logic_error("Reached end of control flow in cached_func");
}
@ -231,19 +180,12 @@ static inline scorer_context cached_init(PyObject* py_str, int def_process, Args
if (str.data == NULL) return {NULL, NULL, NULL};
switch(str.kind){
#if PY_VERSION_HEX >= PYTHON_VERSION(3, 0, 0)
case PyUnicode_1BYTE_KIND:
return get_scorer_context<CachedScorer, uint8_t>(str, def_process, args...);
case PyUnicode_2BYTE_KIND:
return get_scorer_context<CachedScorer, uint16_t>(str, def_process, args...);
case PyUnicode_4BYTE_KIND:
return get_scorer_context<CachedScorer, uint32_t>(str, def_process, args...);
#else
case CHAR_STRING:
return get_scorer_context<CachedScorer, uint8_t>(str, def_process, args...);
case UNICODE_STRING:
return get_scorer_context<CachedScorer, Py_UNICODE>(str, def_process, args...);
#endif
default:
throw std::logic_error("Reached end of control flow in cached_init");
}

293
src/cpp_string_metric.hpp
View File

@ -21,12 +21,6 @@ private:
char const* m_error;
};
#if PY_VERSION_HEX < PYTHON_VERSION(3, 0, 0)
enum StringKind{
CHAR_STRING,
UNICODE_STRING
};
#endif
struct proc_string {
int kind;
@ -38,7 +32,6 @@ static proc_string convert_string(PyObject* py_str)
{
proc_string str = {0, NULL, 0};
#if PY_VERSION_HEX >= PYTHON_VERSION(3, 0, 0)
if (!PyUnicode_Check(py_str)) {
throw PythonTypeError("choice must be a String or None");
}
@ -54,21 +47,6 @@ static proc_string convert_string(PyObject* py_str)
str.kind = PyUnicode_KIND(py_str);
str.data = PyUnicode_DATA(py_str);
str.length = PyUnicode_GET_LENGTH(py_str);
#else /* Python 2 */
if (PyObject_TypeCheck(py_str, &PyString_Type)) {
str.kind = CHAR_STRING;
str.length = PyString_GET_SIZE(py_str);
str.data = (void*)PyString_AS_STRING(py_str);
}
else if (PyObject_TypeCheck(py_str, &PyUnicode_Type)) {
str.kind = UNICODE_STRING;
str.length = PyUnicode_GET_SIZE(py_str);
str.data = (void*)PyUnicode_AS_UNICODE(py_str);
}
else {
throw PythonTypeError("choice must be a String, Unicode or None");
}
#endif
return str;
}
@ -77,7 +55,6 @@ static proc_string convert_string(PyObject* py_str)
* Levenshtein
*/
#if PY_VERSION_HEX >= PYTHON_VERSION(3, 0, 0)
template<typename CharT>
size_t levenshtein_impl_inner(proc_string s1, proc_string s2,
size_t insertion, size_t deletion, size_t substitution, size_t max)
@ -135,61 +112,11 @@ PyObject* levenshtein_impl(PyObject* s1, PyObject* s2,
return PyLong_FromSize_t(result);
}
#else
template<typename CharT>
size_t levenshtein_impl_inner(proc_string s1, proc_string s2,
size_t insertion, size_t deletion, size_t substitution, size_t max)
{
switch(s2.kind){
case CHAR_STRING:
return string_metric::levenshtein(
rapidfuzz::basic_string_view<CharT>((CharT*)s1.data, s1.length),
rapidfuzz::basic_string_view<uint8_t>((uint8_t*)s2.data, s2.length),
{insertion, deletion, substitution}, max
);
default:
return string_metric::levenshtein(
rapidfuzz::basic_string_view<CharT>((CharT*)s1.data, s1.length),
rapidfuzz::basic_string_view<Py_UNICODE>((Py_UNICODE*)s2.data, s2.length),
{insertion, deletion, substitution}, max
);
}
}
PyObject* levenshtein_impl(PyObject* s1, PyObject* s2,
size_t insertion, size_t deletion, size_t substitution, size_t max)
{
size_t result = 0;
proc_string c_s1 = convert_string(s1);
if (c_s1.data == NULL) Py_RETURN_NONE;
proc_string c_s2 = convert_string(s2);
if (c_s2.data == NULL) Py_RETURN_NONE;
switch(c_s1.kind){
case CHAR_STRING:
result = levenshtein_impl_inner<uint8_t>(
c_s1, c_s2, insertion, deletion, substitution, max);
break;
default:
result = levenshtein_impl_inner<Py_UNICODE>(
c_s1, c_s2, insertion, deletion, substitution, max);
break;
}
if (result == (std::size_t)-1) {
return PyLong_FromLong(-1);
}
return PyLong_FromSize_t(result);
}
#endif
/*
* Normalized Levenshtein
*/
#if PY_VERSION_HEX >= PYTHON_VERSION(3, 0, 0)
template<typename CharT>
inline double normalized_levenshtein_impl_inner(proc_string s1, proc_string s2,
size_t insertion, size_t deletion, size_t substitution, double score_cutoff)
@ -237,49 +164,8 @@ double normalized_levenshtein_impl(PyObject* s1, PyObject* s2,
c_s1, c_s2, insertion, deletion, substitution, score_cutoff);
}
}
#else
template<typename CharT>
inline double normalized_levenshtein_impl_inner(proc_string s1, proc_string s2,
size_t insertion, size_t deletion, size_t substitution, double score_cutoff)
{
switch(s2.kind){
case PyUnicode_1BYTE_KIND:
return string_metric::normalized_levenshtein(
rapidfuzz::basic_string_view<CharT>((CharT*)s1.data, s1.length),
rapidfuzz::basic_string_view<uint8_t>((uint8_t*)s2.data, s2.length),
{insertion, deletion, substitution}, score_cutoff
);
default:
return string_metric::normalized_levenshtein(
rapidfuzz::basic_string_view<CharT>((CharT*)s1.data, s1.length),
rapidfuzz::basic_string_view<Py_UNICODE>((Py_UNICODE*)s2.data, s2.length),
{insertion, deletion, substitution}, score_cutoff
);
}
}
double normalized_levenshtein_impl(PyObject* s1, PyObject* s2,
size_t insertion, size_t deletion, size_t substitution, double score_cutoff)
{
proc_string c_s1 = convert_string(s1);
if (c_s1.data == NULL) return 0.0;
proc_string c_s2 = convert_string(s2);
if (c_s2.data == NULL) return 0.0;
switch(c_s1.kind){
case PyUnicode_1BYTE_KIND:
return normalized_levenshtein_impl_inner<uint8_t>(
c_s1, c_s2, insertion, deletion, substitution, score_cutoff);
default:
return normalized_levenshtein_impl_inner<Py_UNICODE>(
c_s1, c_s2, insertion, deletion, substitution, score_cutoff);
}
}
#endif
#if PY_VERSION_HEX >= PYTHON_VERSION(3, 0, 0)
template<typename CharT>
inline double normalized_levenshtein_impl_inner_default_process(proc_string s1, proc_string s2,
size_t insertion, size_t deletion, size_t substitution, double score_cutoff)
@ -337,61 +223,11 @@ double normalized_levenshtein_impl_default_process(PyObject* s1, PyObject* s2,
c_s1, c_s2, insertion, deletion, substitution, score_cutoff);
}
}
#else
template<typename CharT>
inline double normalized_levenshtein_impl_inner_default_process(proc_string s1, proc_string s2,
size_t insertion, size_t deletion, size_t substitution, double score_cutoff)
{
switch(s2.kind){
case PyUnicode_1BYTE_KIND:
return string_metric::levenshtein(
utils::default_process(
rapidfuzz::basic_string_view<CharT>((CharT*)s1.data, s1.length)
),
utils::default_process(
rapidfuzz::basic_string_view<uint8_t>((uint8_t*)s1.data, s1.length)
),
{insertion, deletion, substitution}, score_cutoff
);
default:
return string_metric::levenshtein(
utils::default_process(
rapidfuzz::basic_string_view<CharT>((CharT*)s1.data, s1.length)
),
utils::default_process(
rapidfuzz::basic_string_view<Py_UNICODE>((Py_UNICODE*)s1.data, s1.length)
),
{insertion, deletion, substitution}, score_cutoff
);
}
}
double normalized_levenshtein_impl_default_process(PyObject* s1, PyObject* s2,
size_t insertion, size_t deletion, size_t substitution, double score_cutoff)
{
proc_string c_s1 = convert_string(s1);
if (c_s1.data == NULL) return 0.0;
proc_string c_s2 = convert_string(s2);
if (c_s2.data == NULL) return 0.0;
switch(c_s1.kind){
case PyUnicode_1BYTE_KIND:
return normalized_levenshtein_impl_inner_default_process<uint8_t>(
c_s1, c_s2, insertion, deletion, substitution, score_cutoff);
default:
return normalized_levenshtein_impl_inner_default_process<Py_UNICODE>(
c_s1, c_s2, insertion, deletion, substitution, score_cutoff);
}
}
#endif
/*
* Hamming
*/
#if PY_VERSION_HEX >= PYTHON_VERSION(3, 0, 0)
template<typename CharT>
size_t hamming_impl_inner(proc_string s1, proc_string s2, size_t max)
{
@ -444,57 +280,11 @@ PyObject* hamming_impl(PyObject* s1, PyObject* s2, size_t max)
return PyLong_FromSize_t(result);
}
#else
template<typename CharT>
size_t hamming_impl_inner(proc_string s1, proc_string s2, size_t max)
{
switch(s2.kind){
case CHAR_STRING:
return string_metric::hamming(
rapidfuzz::basic_string_view<CharT>((CharT*)s1.data, s1.length),
rapidfuzz::basic_string_view<uint8_t>((uint8_t*)s2.data, s2.length),
max
);
default:
return string_metric::hamming(
rapidfuzz::basic_string_view<CharT>((CharT*)s1.data, s1.length),
rapidfuzz::basic_string_view<Py_UNICODE>((Py_UNICODE*)s2.data, s2.length),
max
);
}
}
PyObject* hamming_impl(PyObject* s1, PyObject* s2, size_t max)
{
size_t result = 0;
proc_string c_s1 = convert_string(s1);
if (c_s1.data == NULL) Py_RETURN_NONE;
proc_string c_s2 = convert_string(s2);
if (c_s2.data == NULL) Py_RETURN_NONE;
switch(c_s1.kind){
case CHAR_STRING:
result = hamming_impl_inner<uint8_t>(c_s1, c_s2, max);
break;
default:
result = hamming_impl_inner<Py_UNICODE>(c_s1, c_s2, max);
break;
}
if (result == (std::size_t)-1) {
return PyLong_FromLong(-1);
}
return PyLong_FromSize_t(result);
}
#endif
/*
* Normalized Hamming
*/
#if PY_VERSION_HEX >= PYTHON_VERSION(3, 0, 0)
template<typename CharT>
inline double normalized_hamming_impl_inner(proc_string s1, proc_string s2, double score_cutoff)
{
@ -537,45 +327,7 @@ double normalized_hamming_impl(PyObject* s1, PyObject* s2, double score_cutoff)
return normalized_hamming_impl_inner<uint32_t>(c_s1, c_s2, score_cutoff);
}
}
#else
template<typename CharT>
inline double normalized_hamming_impl_inner(proc_string s1, proc_string s2, double score_cutoff)
{
switch(s2.kind){
case PyUnicode_1BYTE_KIND:
return string_metric::normalized_hamming(
rapidfuzz::basic_string_view<CharT>((CharT*)s1.data, s1.length),
rapidfuzz::basic_string_view<uint8_t>((uint8_t*)s2.data, s2.length),
score_cutoff
);
default:
return string_metric::normalized_hamming(
rapidfuzz::basic_string_view<CharT>((CharT*)s1.data, s1.length),
rapidfuzz::basic_string_view<Py_UNICODE>((Py_UNICODE*)s2.data, s2.length),
score_cutoff
);
}
}
double normalized_hamming_impl(PyObject* s1, PyObject* s2, double score_cutoff)
{
proc_string c_s1 = convert_string(s1);
if (c_s1.data == NULL) return 0.0;
proc_string c_s2 = convert_string(s2);
if (c_s2.data == NULL) return 0.0;
switch(c_s1.kind){
case PyUnicode_1BYTE_KIND:
return normalized_hamming_impl_inner<uint8_t>(c_s1, c_s2, score_cutoff);
default:
return normalized_hamming_impl_inner<Py_UNICODE>(c_s1, c_s2, score_cutoff);
}
}
#endif
#if PY_VERSION_HEX >= PYTHON_VERSION(3, 0, 0)
template<typename CharT>
inline double normalized_hamming_impl_inner_default_process(
proc_string s1, proc_string s2, double score_cutoff)
@ -629,48 +381,3 @@ double normalized_hamming_impl_default_process(PyObject* s1, PyObject* s2, doubl
return normalized_hamming_impl_inner_default_process<uint32_t>(c_s1, c_s2, score_cutoff);
}
}
#else
template<typename CharT>
inline double normalized_hamming_impl_inner_default_process(
proc_string s1, proc_string s2, double score_cutoff)
{
switch(s2.kind){
case PyUnicode_1BYTE_KIND:
return string_metric::hamming(
utils::default_process(
rapidfuzz::basic_string_view<CharT>((CharT*)s1.data, s1.length)
),
utils::default_process(
rapidfuzz::basic_string_view<uint8_t>((uint8_t*)s1.data, s1.length)
),
score_cutoff
);
default:
return string_metric::hamming(
utils::default_process(
rapidfuzz::basic_string_view<CharT>((CharT*)s1.data, s1.length)
),
utils::default_process(
rapidfuzz::basic_string_view<Py_UNICODE>((Py_UNICODE*)s1.data, s1.length)
),
score_cutoff
);
}
}
double normalized_hamming_impl_default_process(PyObject* s1, PyObject* s2, double score_cutoff)
{
proc_string c_s1 = convert_string(s1);
if (c_s1.data == NULL) return 0.0;
proc_string c_s2 = convert_string(s2);
if (c_s2.data == NULL) return 0.0;
switch(c_s1.kind){
case PyUnicode_1BYTE_KIND:
return normalized_hamming_impl_inner_default_process<uint8_t>(c_s1, c_s2, score_cutoff);
default:
return normalized_hamming_impl_inner_default_process<Py_UNICODE>(c_s1, c_s2, score_cutoff);
}
}
#endif

23
src/cpp_utils.hpp
View File

@ -19,8 +19,6 @@ private:
char const* m_error;
};
#if PY_VERSION_HEX >= PYTHON_VERSION(3, 0, 0)
PyObject* default_process_impl(PyObject* sentence) {
if (!PyUnicode_Check(sentence)) {
throw PythonTypeError("sentence must be a String");
@ -58,24 +56,3 @@ PyObject* default_process_impl(PyObject* sentence) {
}
}
}
#else
PyObject* default_process_impl(PyObject* sentence) {
if (PyObject_TypeCheck(sentence, &PyString_Type)) {
Py_ssize_t len = PyString_GET_SIZE(sentence);
char* str = PyString_AS_STRING(sentence);
auto proc_str = utils::default_process(rapidfuzz::basic_string_view<char>(str, len));
return PyString_FromStringAndSize(proc_str.data(), proc_str.size());
}
else if (PyObject_TypeCheck(sentence, &PyUnicode_Type)) {
Py_ssize_t len = PyUnicode_GET_SIZE(sentence);
const Py_UNICODE* str = PyUnicode_AS_UNICODE(sentence);
auto proc_str = utils::default_process(rapidfuzz::basic_string_view<Py_UNICODE>(str, len));
return PyUnicode_FromUnicode(proc_str.data(), proc_str.size());
}
else {
throw PythonTypeError("choice must be a String or Unicode");
}
}
#endif