Merge pull request #1 from iamlostshe/master

Many updates

Author: FlacSy

.gitignore

@@ -1,3 +1,7 @@
 .libtest
 .vscode
-__pycache__/
+.python-version
+__pycache__/
+.ruff_cache
+.mypy_cache
+*test*

README.md

@@ -41,24 +41,34 @@ pip3 install git+https://github.com/FlacSy/badwords.git
 ### Инициализация
 
 ```python
-ProfanityFilter(languages: List[str] = None, all_languages: bool = False)
+p = ProfanityFilter()
+
+p.init(languages: List[str] | None = None)
 ```
 
 #### Параметры
 
-- `languages` (список строк, необязательно): Список языков, для которых будут загружены слова нецензурной лексики. Если не указано, будут использованы все доступные языки при установке флага `all_languages` в `True`.
-- `all_languages` (логическое значение, необязательно): Флаг для загрузки слов нецензурной лексики для всех доступных языков. По умолчанию `False`.
+- `languages` (список строк, необязательно): Список языков, для которых будут загружены слова нецензурной лексики. Если не указано, будут использованы все доступные языки.
 
 ### Примеры использования
 
 ```python
+import asyncio
+
 from badwords import ProfanityFilter
 
-# Инициализация с использованием английского и испанского языков
-my_filter = ProfanityFilter(languages=['en', 'sp'])
 
-# Инициализация с использованием всех доступных языков
-my_filter_all = ProfanityFilter(all_languages=True)
+async def main() -> None:
+    # Инициализация с использованием английского и испанского языков
+    _filter = ProfanityFilter()
+    await _filter.init(["en", "sp"])
+
+    # Инициализация с использованием всех доступных языков
+    await _filter.init()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
 ```
 
 ### Методы
@@ -74,7 +84,7 @@ my_filter_all = ProfanityFilter(all_languages=True)
 ##### Пример
 
 ```python
-language_files = my_filter.initialize_language_files()
+language_files = await _filter.initialize_language_files()
 print(language_files)
 ```
 
@@ -89,25 +99,10 @@ print(language_files)
 ##### Пример
 
 ```python
-bad_words = my_filter.initialize_bad_words()
+bad_words = await _filter.initialize_bad_words()
 print(bad_words)
 ```
 
-#### `compile_patterns()`
-
-Компиляция регулярных выражений для слов нецензурной лексики.
-
-##### Возвращаемое значение
-
-- Словарь, который сопоставляет имена языков с компилированными регулярными выражениями.
-
-##### Пример
-
-```python
-patterns = my_filter.compile_patterns()
-print(patterns)
-```
-
 #### `add_words(words: List[str])`
 
 Добавление пользовательских слов нецензурной лексики в фильтр.
@@ -119,7 +114,7 @@ print(patterns)
 ##### Пример
 
 ```python
-my_filter.add_words(['customword1', 'customword2'])
+await _filter.add_words(["customword1", "customword2"])
 ```
 
 #### `similar(a: str, b: str)`
@@ -153,11 +148,11 @@ my_filter.add_words(['customword1', 'customword2'])
 
 ```python
 # Проверка на наличие нецензурной лексики
-contains_profanity = my_filter.filter_text('This is some bad text', match_threshold=0.9)
+contains_profanity = await _filter.filter_text("This is some bad text", match_threshold=0.9)
 print(contains_profanity)  # True или False
 
 # Проверка на наличие нецензурной лексики с заменой
-filtered_text = my_filter.filter_text('This is some bad text', replace_character='*')
+filtered_text = await _filter.filter_text("This is some bad text", replace_character="*")
 print(filtered_text)  # Текст с заменёнными непристойными словами
 ```
 
@@ -172,8 +167,8 @@ print(filtered_text)  # Текст с заменёнными непристой
 ##### Пример
 
 ```python
-all_languages = my_filter.get_all_languages()
-print(all_languages)  # ['en', 'sp', 'fr', 'de', ...]
+all_languages = await _filter.get_all_languages()
+print(all_languages)  # ["en", "sp", "fr", "de", ...]
 ```
 
 ## Поддерживаемые языки
@@ -209,16 +204,26 @@ print(all_languages)  # ['en', 'sp', 'fr', 'de', ...]
 ## Полный пример использования
 
 ```python
+import asyncio
+
 from badwords import ProfanityFilter
 
-# Создаем экземпляр фильтра, указывая нужные языки
-my_filter = ProfanityFilter(languages=['en', 'sp'])
+
+async def main() -> None:
+    # Создаем экземпляр фильтра, указывая нужные языки
+    _filter = ProfanityFilter()
+    await _filter.init(["en", "sp"])
+
+    text ="Text with inappropriate words"
+
+    await check_profanity(_filter, text)
+    await check_profanity_with_replace(_filter, text)
 
 # Функция для проверки текста на наличие нецензурной лексики
-def check_profanity(text: str):
-    result = my_filter.filter_text(
+async def check_profanity(_filter: ProfanityFilter, text: str) -> None:
+    result = await _filter.filter_text(
         text=text,
-        match_threshold=0.9
+        match_threshold=0.9,
     )
 
     if result:
@@ -227,16 +232,15 @@ def check_profanity(text: str):
         print("Этот текст не содержит нецензурной лексики.")
 
 # Функция для проверки текста на наличие нецензурной лексики с заменой
-def check_profanity_with_replace(text: str):
-    result = my_filter.filter_text(
+async def check_profanity_with_replace(_filter: ProfanityFilter, text: str) -> str:
+    result = await _filter.filter_text(
         text=text,
         match_threshold=0.8,
-        replace_character="*"
+        replace_character="*",
     )
 
     print(result)
 
 if __name__ == "__main__":
-    check_profanity("Text with inappropriate words")
-    check_profanity_with_replace("Text with inappropriate words")
-```
+    asyncio.run(main())
+```

badwords/__init__.py

@@ -1,3 +1,5 @@
+"""A library for effective moderation of content."""
+
 from .check import ProfanityFilter
 
-__all__ = ['ProfanityFilter']
+__all__ = ["ProfanityFilter"]

badwords/check.py

@@ -1,102 +1,108 @@
-import os
-import re
-from typing import List, Dict, Set
+"""Module for checking text for badwords."""
+
+from __future__ import annotations
+
 from difflib import SequenceMatcher
+from pathlib import Path
+from typing import Self
+
+from .exceptions import NotSupportedLanguage
+
 
 class ProfanityFilter:
-    """
-    A class for filtering profanity from text.
-    """
+    """A class for filtering profanity from text."""
 
-    def __init__(self, languages: List[str] = None, all_languages: bool = False):
-        """
-        Initialize the profanity filter.
+    async def init(self: Self,
+            languages: list[str] | None = None,
+        ) -> None:
+        """Initialize the profanity filter.
 
         :param languages: List of languages to load profanity words for.
         :param all_languages: Flag to load profanity words for all available languages.
         """
-        self.script_dir = os.path.dirname(os.path.realpath(__file__))
-        self.language_files: Dict[str, str] = self.initialize_language_files()
-        self.languages = languages or list(self.language_files.keys()) if all_languages else languages
-        self.bad_words: Dict[str, Set[str]] = self.initialize_bad_words()
-        self.patterns: Dict[str, re.Pattern] = self.compile_patterns()
-        self.custom_bad_words: Set[str] = set()
-
-    def initialize_language_files(self) -> Dict[str, str]:
-        """
-        Initialize language files.
+        self.resource_dir = Path(__file__).parent / "resource"
+
+        self.language_files = await self.initialize_language_files()
+
+        if languages:
+            if all(i in self.language_files for i in languages):
+                self.language_files = languages
+            else:
+                raise NotSupportedLanguage
+
+        self.bad_words = await self.initialize_bad_words()
+
+    async def initialize_language_files(self: Self) -> list[str]:
+        """Initialize language files.
 
         :return: Dictionary mapping language names to file paths.
         """
-        resource_dir = os.path.join(self.script_dir, 'resource')
-        return {os.path.splitext(filename)[0]: os.path.join(resource_dir, filename) for filename in os.listdir(resource_dir)}
+        return [str(path)[-6:-4] for path in (self.resource_dir).iterdir()]
 
-    def initialize_bad_words(self) -> Dict[str, Set[str]]:
-        """
-        Initialize profanity words for each language.
+    async def initialize_bad_words(self: Self) -> set[str]:
+        """Initialize profanity words for each language.
 
         :return: Dictionary mapping language names to sets of profanity words.
         """
-        bad_words = {}
-        for language in self.languages:
-            file_path = self.language_files.get(language)
-            if file_path:
-                with open(file_path, 'r', encoding='utf-8') as file:
-                    bad_words[language] = {line.strip() for line in file}
-        return bad_words
+        bad_words = set()
 
-    def compile_patterns(self) -> Dict[str, re.Pattern]:
-        """
-        Compile regular expression patterns for profanity words.
+        for lang in self.language_files:
+            with (self.resource_dir / f"{lang}.bdw").open(encoding="utf-8") as f:
+                bad_words.update(f.read().split())
 
-        :return: Dictionary mapping language names to compiled regex patterns.
-        """
-        return {language: re.compile(r'\b(?:' + '|'.join(map(re.escape, words)) + r')\b', re.IGNORECASE) for language, words in self.bad_words.items()}
+        return bad_words
 
-    def add_words(self, words: List[str]):
-        """
-        Add custom profanity words to the filter.
+    async def add_words(self: Self, words: list[str]) -> None:
+        """Add custom profanity words to the filter.
 
         :param words: List of custom profanity words.
         """
-        self.custom_bad_words.update(words)
+        self.bad_words.update(words)
 
-    def similar(self, a: str, b: str) -> float:
-        """
-        Compute similarity ratio between two strings.
+    async def similar(self: Self, a: str, b: str) -> float:
+        """Compute similarity ratio between two strings.
 
         :param a: First string.
         :param b: Second string.
         :return: Similarity ratio.
         """
         return SequenceMatcher(None, a, b).ratio()
 
-    def filter_text(self, text: str, match_threshold: float = 0.8, replace_character=None):
-        """
-        Check if the given text contains profanity.
+    async def filter_text(
+            self: Self, text: str,
+            match_threshold: float | None = None,
+            replace_character: str | None = None,
+        ) -> bool | str:
+        """Check if the given text contains profanity.
 
         :param text: Input text to check.
         :param match_threshold: Threshold for similarity match.
-        :param replace_character: Character to replace profane words with. If None, return True/False.
-        :return: True if profanity found, False otherwise. If replace_character is specified, return filtered text.
+        :param replace_character: Character to replace profane words with. If None,
+            return True/False.
+        :return: True if profanity found, False otherwise. If replace_character is
+            specified, return filtered text.
         """
-        all_bad_words = set.union(self.custom_bad_words, *self.bad_words.values())
-
-        words_in_text = text.lower().split(' ')
-        filtered_text = text.lower()
-        for word in words_in_text:
-            for bad_word in all_bad_words:
-                if self.similar(word, bad_word) > match_threshold:
-                    if replace_character is not None:
-                        filtered_text = filtered_text.replace(word, replace_character * len(word))
-                    else:
-                        return True if replace_character is None else filtered_text
-        return False if replace_character is None else filtered_text
-
-    def get_all_languages(self) -> List[str]:
-        """
-        Get a list of all available languages.
+        if not match_threshold:
+            match_threshold = 1
+
+        text = text.lower()
+
+        for word in text.split():
+            if word in self.bad_words:
+                return True
+
+            if 0 < match_threshold < 1:
+                for bad_word in self.bad_words:
+                    if await self.similar(word, bad_word) > match_threshold:
+                        if replace_character:
+                            return text.replace(word)
+                        return True
+
+        return False
+
+    async def get_all_languages(self: Self) -> list[str]:
+        """Get a list of all available languages.
 
         :return: List of all language names.
         """
-        return list(self.language_files.keys())
+        return self.language_files

badwords/exceptions.py

@@ -0,0 +1,11 @@
+"""Exceptions module."""
+
+from typing import Self
+
+
+class NotSupportedLanguage(BaseException):
+    """Unsupport language check."""
+
+    def __str__(self: Self) -> str:
+        """String-like representation of exception."""
+        return "This language is not supported"