72 lines
2.9 KiB
Python
72 lines
2.9 KiB
Python
import functools
|
|
import time
|
|
import typing
|
|
from collections.abc import Callable, Iterable, Iterator
|
|
|
|
import Levenshtein
|
|
from folkugat_web.config import search as config
|
|
from folkugat_web.dal.sql import get_connection
|
|
from folkugat_web.dal.sql.temes import query as temes_q
|
|
from folkugat_web.log import logger
|
|
from folkugat_web.model import search as search_model
|
|
from folkugat_web.model import temes as model
|
|
|
|
|
|
def get_query_word_similarity(query_word: str, text_ngrams: model.NGrams) -> search_model.SearchMatch:
|
|
n = len(query_word)
|
|
if n < config.MIN_NGRAM_LENGTH:
|
|
return search_model.SearchMatch(distance=0.0, ngram='')
|
|
ns = filter(lambda i: i >= config.MIN_NGRAM_LENGTH, range(
|
|
n - config.QUERY_NGRAM_RANGE, n + config.QUERY_NGRAM_RANGE + 1))
|
|
candidate_ngrams = ((m, ngram)
|
|
for m, ngrams in map(lambda i: (i, text_ngrams.get(i, [])), ns)
|
|
for ngram in ngrams)
|
|
return min((search_model.SearchMatch(distance=Levenshtein.distance(query_word, ngram)/m,
|
|
ngram=ngram)
|
|
for m, ngram in candidate_ngrams),
|
|
default=search_model.SearchMatch(distance=float("inf"), ngram=""))
|
|
|
|
|
|
def get_query_similarity(query: str, ngrams: model.NGrams) -> search_model.SearchMatch:
|
|
query_words = query.lower().split()
|
|
word_matches = map(lambda query_word: get_query_word_similarity(query_word, ngrams), query_words)
|
|
return search_model.SearchMatch.combine_matches(word_matches)
|
|
|
|
|
|
def build_result(query: str, entry: tuple[int, model.NGrams]) -> search_model.QueryResult:
|
|
if len(query) == 0:
|
|
return search_model.QueryResult(
|
|
id=entry[0],
|
|
distance=0,
|
|
ngram="",
|
|
)
|
|
match = get_query_similarity(query, entry[1])
|
|
return search_model.QueryResult(
|
|
id=entry[0],
|
|
distance=match.distance,
|
|
ngram=match.ngram,
|
|
)
|
|
|
|
|
|
T = typing.TypeVar("T")
|
|
|
|
|
|
def _thread(it: Iterable[T], *funcs: Callable[[Iterable], Iterable]) -> Iterable:
|
|
return functools.reduce(lambda i, fn: fn(i), funcs, it)
|
|
|
|
|
|
def busca_temes(query: str, hidden: bool = False, limit: int = 20, offset: int = 0) -> list[model.Tema]:
|
|
t0 = time.time()
|
|
with get_connection() as con:
|
|
result = _thread(
|
|
temes_q.get_tema_id_to_ngrams(con).items(),
|
|
lambda tema_id_to_ngrams: (build_result(query, entry) for entry in tema_id_to_ngrams),
|
|
lambda results: filter(lambda qr: qr.distance <= config.SEARCH_DISTANCE_THRESHOLD, results),
|
|
lambda results: sorted(results, key=lambda qr: qr.distance),
|
|
lambda results: filter(None, map(lambda qr: temes_q.get_tema_by_id(qr.id, con), results)),
|
|
lambda results: filter(lambda t: hidden or not t.hidden, results),
|
|
)
|
|
result = list(result)[offset:offset + limit]
|
|
logger.info(f"Search time: { int((time.time() - t0) * 1000) } ms")
|
|
return result
|