match.py

from sentence_transformers import util
import jellyfish
import Levenshtein

def compute_similarity_embedding(list1, list2, threshold=0.52, model=None):
    if not list1 or not list2:
        return []
    white_list = []
    
    # Data validation to check types in list2
    for i, item in enumerate(list2):
        if not isinstance(item, str):
            print(f"ERROR: list2 contains non-string item at index {i}: {type(item)} - {item}")
            # Replace non-string items with empty string to avoid errors
            list2[i] = str(item)
    
    # Generate sentence embeddings for both lists
    try:
        embeddings1 = model.encode(list1, convert_to_tensor=True)
        embeddings2 = model.encode(list2, convert_to_tensor=True)
    except Exception as e:
        print(f"ERROR encoding items: {e}")
        print(f"list1: {list1}")
        print(f"list2: {list2}")
        return []
    
    # Calculate cosine similarity
    cosine_scores = util.pytorch_cos_sim(embeddings1, embeddings2)

    for i, item1 in enumerate(list1):
        for j, item2 in enumerate(list2):
            similarity = cosine_scores[i][j].item()
            print(f"Embedding - item1: {item1}, item2: {item2}, similarity: {similarity}")
            if similarity >= threshold:
                white_list.append(item1)
                break
    return white_list

def compute_similarity_jaro_winkler(list1, list2, threshold=0.82):
    white_list = []
    for item1 in list1:
        for item2 in list2:
            similarity = jellyfish.jaro_winkler_similarity(item1.lower(), item2.lower())
            print(f"Jaro-Winkler - item1: {item1}, item2: {item2}, similarity: {similarity}")
            if similarity >= threshold:
                white_list.append(item1)
                break
    return white_list

def compute_similarity_levenshtein(list1, list2, threshold=0.50):
    white_list = []
    for item1 in list1:
        for item2 in list2:
            # Calculate Levenshtein distance
            distance = Levenshtein.distance(item1.lower(), item2.lower())
            
            # Normalize by the length of the longer string
            max_len = max(len(item1), len(item2))
            if max_len == 0:  # Handle empty strings
                similarity = 1.0
            else:
                similarity = 1.0 - (distance / max_len)
                
            print(f"Levenshtein - item1: {item1}, item2: {item2}, similarity: {similarity}")
            if similarity >= threshold:
                white_list.append(item1)
                break
    return white_list