Search by properties

folkugat_web/api/sessio/index.py

@@ -197,28 +197,3 @@ def set_tema_new(
         entry_id=entry_id,
         tema_id=new_tema.id,
     )
-
-
-# @router.get("/api/sessio/{session_id}/set/{set_id}/score")
-# async def render(
-#     request: Request,
-#     _: auth.RequireLogin,
-#     session_id: int,
-#     set_id: int,
-# ):
-#     set_entry = playlists_service.get_set(session_id=session_id, set_id=set_id)
-#     if not set_entry:
-#         raise HTTPException(status_code=404, detail="Could not find set!")
-#     set_entry = playlists_service.add_temes_to_set(set_entry)
-#     tune_set = lilypond_build.set_from_set(set_entry=set_entry)
-#     set_source = lilypond_source.set_source(tune_set=tune_set)
-#     pdf_result = await lilypond_render.render(
-#         source=set_source,
-#         output=lilypond_render.RenderOutput.PDF,
-#     )
-#     if output_filename := pdf_result.result:
-#         score_render_url = files.get_db_file_path(output_filename)
-#         # return temes.score_render(request=request, score_id=score_id, score_render_url=score_render_url)
-#         return HTMLResponse(content=score_render_url)
-#     else:
-#         return HTMLResponse()

folkugat_web/api/sessio/live.py

@@ -22,16 +22,16 @@ def page(
 
 
 @router.get("/api/content/live")
-def contingut(
+async def contingut(
     request: Request,
     logged_in: auth.LoggedIn,
 ):
-    return set_page.live(request, logged_in)
+    return await set_page.live(request, logged_in)
 
 
 @router.get("/api/content/live/set")
-def get_set_page(
+async def get_set_page(
     request: Request,
     logged_in: auth.LoggedIn,
 ):
-    return set_page.live_set(request, logged_in)
+    return await set_page.live_set(request, logged_in)

folkugat_web/api/temes/index.py

@@ -1,3 +1,4 @@
+import urllib.parse
 from typing import Annotated
 
 from fastapi import Request
@@ -13,13 +14,16 @@ def page(
         request: Request,
         logged_in: auth.LoggedIn,
         query: Annotated[str, Param()] = "",
+        properties: Annotated[list[str] | None, Param()] = None,
 ):
+    properties = properties or []
+    content_url = f"/api/content/temes?{temes.build_temes_params(query=query, properties=properties)}"
     return templates.TemplateResponse(
         "index.html",
         {
             "request": request,
             "page_title": "Folkugat",
-            "content": f"/api/content/temes?query={query}",
+            "content": content_url,
             "logged_in": logged_in,
             "animate": False,
         }
@@ -31,10 +35,31 @@ def content(
     request: Request,
     logged_in: auth.LoggedIn,
     query: Annotated[str, Param()] = "",
+    properties: Annotated[list[str] | None, Param()] = None,
 ):
-    return temes.temes_pagina(request, logged_in, query)
+    properties = properties or []
+    return temes.temes_pagina(
+        request=request,
+        logged_in=logged_in,
+        query=query,
+        properties=properties,
+    )
 
 
 @router.get("/api/temes/busca")
-def busca(request: Request, query: str, logged_in: auth.LoggedIn, limit: int = 10, offset: int = 0):
+def busca(
-    return temes.temes_busca(request, query=query, limit=limit, offset=offset, logged_in=logged_in)
+    request: Request,
+    logged_in: auth.LoggedIn,
+    query: Annotated[str, Param()],
+    properties: Annotated[list[str] | None, Param()] = None,
+    limit: int = 10,
+    offset: int = 0,
+):
+    return temes.temes_busca(
+        request=request,
+        query=query,
+        properties=properties or [],
+        limit=limit,
+        offset=offset,
+        logged_in=logged_in,
+    )

folkugat_web/assets/static/css/main.css

@@ -649,6 +649,11 @@ video {
   margin-bottom: 0.75rem;
 }
 
+.my-4 {
+  margin-top: 1rem;
+  margin-bottom: 1rem;
+}
+
 .mb-3 {
   margin-bottom: 0.75rem;
 }
@@ -944,11 +949,6 @@ video {
   padding: 1rem;
 }
 
-.px-1 {
-  padding-left: 0.25rem;
-  padding-right: 0.25rem;
-}
-
 .px-10 {
   padding-left: 2.5rem;
   padding-right: 2.5rem;

folkugat_web/assets/templates/fragments/temes/pagina.html

@@ -11,6 +11,7 @@
          hx-get="/api/temes/busca"
          hx-trigger="revealed, keyup delay:500ms changed"
          hx-target="#search-results"
+         hx-vars="properties:{{ properties_str }}"
          hx-swap="outerHTML">
   <div id="search-results" class="flex-col"></div>
 </div>

folkugat_web/assets/templates/fragments/temes/result.html

@@ -0,0 +1,34 @@
+<li class="flex flex-col
+           items-center
+          m-4 rounded-lg
+          bg-white">
+  <div class="py-2 px-4 text-brown font-bold">
+    <a href="/tema/{{ tema.id }}">
+      {{ tema.title }}
+    </a>
+  </div>
+  {% if tema.main_score() and tema.main_score().preview_url %}
+    <a href="/tema/{{ tema.id }}"
+       class="max-w">
+    <img class="p-2 max-w"
+        src="{{ tema.main_score().preview_url }}" />
+    </a>
+  {% endif %}
+  {% if tema.properties %}
+  <ul class="flex flex-wrap text-sm
+            py-2 px-4">
+    {% for property in tema.properties %}
+    <button class="bg-beige text-white rounded
+                   m-1 px-2"
+            hx-get="/api/content/temes"
+            hx-target="#content"
+            hx-include="[name=query]"
+            hx-vars="properties:{{ add_property_str(property.value) }}"
+            hx-swap="innerHTML"
+            >
+      {{ property.value }}
+    </button>
+    {% endfor %}
+  </ul>
+  {% endif %}
+</li>

folkugat_web/assets/templates/fragments/temes/result_links.html

@@ -1,19 +0,0 @@
-<ul class="flex flex-row items-center">
-  <li class="px-2 text-beige">
-  {% if tema.has_score %}
-    <a href="/tema/{{ tema.id }}" target="_blank">
-      <i class="fa fa-music px-1" aria-hidden="true"></i>
-    </a>
-  {% endif %}
-  {% if tema.has_lyrics %}
-    <a href="/tema/{{ tema.id }}" target="_blank">
-      <i class="fa fa-font px-1" aria-hidden="true"></i>
-    </a>
-  {% endif %}
-  {% if tema.has_audio %}
-    <a href="/tema/{{ tema.id }}" target="_blank">
-      <i class="fa fa-volume-up px-1" aria-hidden="true"></i>
-    </a>
-  {% endif %}
-  </li>
-</ul>

folkugat_web/assets/templates/fragments/temes/results.html

@@ -12,32 +12,41 @@
       <i>{{ query }}</i>
     </button>
   {% endif %}
-  <ul class="text-left min-w-full w-full">
+  <ul class="flex flex-wrap justify-center my-4">
-    {% for tema in temes %}
+    {% for property in properties %}
-    <li class="flex flex-col
+      <li>
-               m-4 rounded-lg
+        <button class="bg-beige rounded
-               bg-white">
+                       text-white
-      <div class="py-2 px-4 text-brown font-bold">
+                       m-1 px-2"
-        <a href="/tema/{{ tema.id }}">
+                 hx-get="/api/content/temes"
-          {{ tema.title }}
+                 hx-target="#content"
-        </a>
+                 hx-include="[name=query]"
-      </div>
+                 hx-vars="properties:{{ remove_property_str(property) }}"
-      {% if tema.main_score() and tema.main_score().preview_url %}
+                 hx-swap="innerHTML"
-        <img class="p-2 max-w"
+                 >
-             src="{{ tema.main_score().preview_url }}" />
+          {{ property }}
-      {% endif %}
+        </button>
-      {% if tema.properties %}
+      </li>
-      <ul class="flex flex-wrap text-sm
+    {% endfor %}
-                 py-2 px-4">
+    {% for property in property_results %}
-        {% for property in tema.properties %}
+      <li>
-        <div class="bg-beige text-white rounded
+        <button class="border border-beige  rounded
-                    m-1 px-2">
+                       text-white
-          {{ property.value }}
+                       m-1 px-2"
-        </div>
+                 hx-get="/api/content/temes"
+                 hx-target="#content"
+                 hx-vars="properties:{{ add_property_str(property) }}"
+                 hx-swap="innerHTML"
+                 >
+          {{ property }}
+        </button>
+      </li>
     {% endfor %}
   </ul>
-      {% endif %}
+  <hr class="h-px mt-1 mb-3 bg-beige border-0">
-    </li>
+  <ul class="min-w-full w-full">
+    {% for tema in temes %}
+      {% include "fragments/temes/result.html" %}
     {% endfor %}
   </ul>
   {% if prev_offset is not none or next_offset is not none %}

folkugat_web/dal/sql/temes/properties.py

@@ -2,7 +2,9 @@ from collections.abc import Iterable
 from typing import TypedDict
 
 from folkugat_web.dal.sql import Connection, get_connection
+from folkugat_web.model import search as search_model
 from folkugat_web.model import temes as model
+from folkugat_web.services import ngrams
 
 PropertyRowTuple = tuple[int, int, str, str]
 
@@ -86,6 +88,7 @@ def insert_property(property: model.Property, con: Connection | None = None) ->
         cur = con.cursor()
         _ = cur.execute(query, data)
         row: PropertyRowTuple = cur.fetchone()
+    evict_property_value_to_ngrams_cache()
     return row_to_property(row)
 
 
@@ -111,6 +114,7 @@ def update_property(property: model.Property, con: Connection | None = None):
     with get_connection(con) as con:
         cur = con.cursor()
         _ = cur.execute(query, data)
+    evict_property_value_to_ngrams_cache()
 
 
 def delete_property(property_id: int, tema_id: int | None = None, con: Connection | None = None):
@@ -125,3 +129,32 @@ def delete_property(property_id: int, tema_id: int | None = None, con: Connectio
     with get_connection(con) as con:
         cur = con.cursor()
         _ = cur.execute(query, data)
+    evict_property_value_to_ngrams_cache()
+
+
+_property_value_to_ngrams_cache: dict[str, search_model.NGrams] | None = None
+
+
+def evict_property_value_to_ngrams_cache():
+    global _property_value_to_ngrams_cache
+    _property_value_to_ngrams_cache = None
+
+
+def get_property_value_to_ngrams(con: Connection | None = None) -> dict[str, search_model.NGrams]:
+    global _property_value_to_ngrams_cache
+    if _property_value_to_ngrams_cache is None:
+        _property_value_to_ngrams_cache = _get_property_value_to_ngrams(con)
+    return _property_value_to_ngrams_cache
+
+
+def _get_property_value_to_ngrams(con: Connection | None = None) -> dict[str, search_model.NGrams]:
+    query = """
+      SELECT value
+      FROM tema_properties
+    """
+    with get_connection(con) as con:
+        cur = con.cursor()
+        _ = cur.execute(query)
+        rows: list[tuple[str]] = cur.fetchall()
+        props = {prop[0] for prop in rows}
+        return {prop: ngrams.get_text_ngrams(prop.lower()) for prop in props}

folkugat_web/fragments/sessio.py

@@ -128,6 +128,7 @@ def busca_tema(
 ):
     results = search_service.busca_temes(
         query=query,
+        properties=[],
         hidden=True,
         limit=4,
         offset=0,

folkugat_web/fragments/set_page.py

@@ -29,13 +29,14 @@ async def pagina(request: Request, session_id: int, set_id: int, logged_in: bool
     )
 
 
-def live(request: Request, logged_in: bool):
+async def live(request: Request, logged_in: bool):
     session = sessions_service.get_live_session()
     set_ = None
     if session and session.id:
         playlist = playlists_service.get_playlist(session_id=session.id)
         if playlist.sets:
             set_ = playlists_service.add_temes_to_set(playlist.sets[-1])
+            set_ = await playlists_service.add_set_score_to_set(set_)
     return templates.TemplateResponse(
         "fragments/sessio/set/pagina.html",
         {
@@ -51,13 +52,14 @@ def live(request: Request, logged_in: bool):
     )
 
 
-def live_set(request: Request, logged_in: bool):
+async def live_set(request: Request, logged_in: bool):
     session = sessions_service.get_live_session()
     set_ = None
     if session and session.id:
         playlist = playlists_service.get_playlist(session_id=session.id)
         if playlist.sets:
             set_ = playlists_service.add_temes_to_set(playlist.sets[-1])
+            set_ = await playlists_service.add_set_score_to_set(set_)
     return templates.TemplateResponse(
         "fragments/sessio/set/set_page.html",
         {

folkugat_web/fragments/temes.py

@@ -1,9 +1,11 @@
+import json
+import urllib.parse
+
 from fastapi import Request
 from folkugat_web.model import temes as model
 from folkugat_web.model.lilypond.processing import RenderError
 from folkugat_web.model.pagines import Pages
 from folkugat_web.services import sessions as sessions_service
-from folkugat_web.services.temes import properties as properties_service
 from folkugat_web.services.temes import query as temes_q
 from folkugat_web.services.temes import scores as scores_service
 from folkugat_web.services.temes import search as temes_s
@@ -11,22 +13,46 @@ from folkugat_web.templates import templates
 from folkugat_web.utils import FnChain
 
 
-def temes_pagina(request: Request, logged_in: bool, query: str):
+def build_temes_params(query: str, properties: list[str]) -> str:
+    content_params = [
+        ("query", query),
+        *[("properties", prop) for prop in properties or []]
+    ]
+    return urllib.parse.urlencode(content_params)
+
+
+def build_property_str(properties: list[str]) -> str:
+    return json.dumps(properties).replace("'", "\\'").replace('"', "'")
+
+
+def temes_pagina(request: Request, logged_in: bool, query: str, properties: list[str]):
+    properties_str = build_property_str(properties)
     return templates.TemplateResponse(
         "fragments/temes/pagina.html",
         {
             "request": request,
             "logged_in": logged_in,
             "query": query,
+            "properties": properties,
+            "properties_str": properties_str,
+            "property_results": [],
             "Pages": Pages,
             "menu_selected_id": Pages.Temes,
         }
     )
 
 
-def temes_busca(request: Request, logged_in: bool, query: str, offset: int = 0, limit: int = 10):
+def temes_busca(
+    request: Request,
+    logged_in: bool,
+    query: str,
+    properties: list[str],
+    offset: int = 0,
+    limit: int = 10,
+):
     temes = temes_s.busca_temes(
         query=query,
+        properties=properties,
         hidden=logged_in,
         limit=limit + 1,
         offset=offset,
@@ -42,11 +68,26 @@ def temes_busca(request: Request, logged_in: bool, query: str, offset: int = 0,
     temes = (
         FnChain.transform(temes) |
         temes_q.temes_compute_stats |
-        properties_service.add_properties_to_temes |
         scores_service.add_scores_to_temes |
+        # No properties added because search already does that
         list
     ).result()
 
+    property_results = [
+        prop for prop in temes_s.busca_properties(query=query, limit=5)
+        if prop not in properties
+    ]
+
+    temes_url = f"/temes?{build_temes_params(query, properties)}"
+
+    def _add_property_str(prop: str) -> str:
+        properties_dedup = list(set(properties + [prop]))
+        return build_property_str(properties_dedup)
+
+    def _remove_property_str(prop: str) -> str:
+        properties_clean = [p for p in properties if p.lower() != prop.lower()]
+        return build_property_str(properties_clean)
+
     return templates.TemplateResponse(
         "fragments/temes/results.html",
         {
@@ -54,11 +95,16 @@ def temes_busca(request: Request, logged_in: bool, query: str, offset: int = 0,
             "logged_in": logged_in,
             "temes": temes,
             "query": query,
+            "properties": properties,
+            "property_results": property_results,
             "prev_offset": prev_offset,
             "next_offset": next_offset,
             "LinkType": model.LinkType,
             "ContentType": model.ContentType,
-        }
+            "add_property_str": _add_property_str,
+            "remove_property_str": _remove_property_str,
+        },
+        headers={"HX-Replace-Url": temes_url},
     )
 
 

folkugat_web/model/search.py

@@ -1,7 +1,8 @@
 import dataclasses
 from collections.abc import Iterable
-from typing import Self
+from typing import Generic, Self, TypeVar
 
+T = TypeVar("T")
 NGrams = dict[int, list[str]]
 
 
@@ -20,7 +21,7 @@ class SearchMatch:
 
 
 @dataclasses.dataclass
-class QueryResult:
+class QueryResult(Generic[T]):
-    id: int
+    result: T
     distance: float
     ngram: str

folkugat_web/services/temes/search.py

@@ -1,17 +1,21 @@
 import time
 from collections.abc import Iterable, Iterator
 from sqlite3 import Connection
-from typing import Callable
+from typing import Callable, TypeVar
 
 import Levenshtein
 from folkugat_web.config import search as config
 from folkugat_web.dal.sql import get_connection
+from folkugat_web.dal.sql.temes import properties as properties_dal
 from folkugat_web.dal.sql.temes import query as temes_q
 from folkugat_web.log import logger
 from folkugat_web.model import search as search_model
 from folkugat_web.model import temes as model
+from folkugat_web.services.temes import properties as properties_service
 from folkugat_web.utils import FnChain
 
+T = TypeVar("T")
+
 
 def get_query_word_similarity(query_word: str, text_ngrams: search_model.NGrams) -> search_model.SearchMatch:
     n = len(query_word)
@@ -34,39 +38,41 @@ def get_query_similarity(query: str, ngrams: search_model.NGrams) -> search_mode
     return search_model.SearchMatch.combine_matches(word_matches)
 
 
-def _build_results_fn(query: str) -> Callable[[Iterable[tuple[int, search_model.NGrams]]],
+def _build_results_fn(query: str) -> Callable[[Iterable[tuple[T, search_model.NGrams]]],
-                                              Iterator[search_model.QueryResult]]:
+                                              Iterator[search_model.QueryResult[T]]]:
-    def build_result(entry: tuple[int, search_model.NGrams]) -> search_model.QueryResult:
+    def build_result(entry: tuple[T, search_model.NGrams]) -> search_model.QueryResult[T]:
         if len(query) == 0:
             return search_model.QueryResult(
-                id=entry[0],
+                result=entry[0],
                 distance=0,
                 ngram="",
             )
         match = get_query_similarity(query, entry[1])
         return search_model.QueryResult(
-            id=entry[0],
+            result=entry[0],
             distance=match.distance,
             ngram=match.ngram,
         )
 
-    def build_results(entries: Iterable[tuple[int, search_model.NGrams]]) -> Iterator[search_model.QueryResult]:
+    def build_results(entries: Iterable[tuple[T, search_model.NGrams]]) -> Iterator[search_model.QueryResult[T]]:
         return map(build_result, entries)
 
     return build_results
 
 
-def _filter_distance(qrs: Iterable[search_model.QueryResult]) -> Iterator[search_model.QueryResult]:
+def _filter_distance(qrs: Iterable[search_model.QueryResult[T]]) -> Iterator[search_model.QueryResult[T]]:
     return filter(lambda qr: qr.distance <= config.SEARCH_DISTANCE_THRESHOLD, qrs)
 
 
-def _sort_by_distance(qrs: Iterable[search_model.QueryResult]) -> list[search_model.QueryResult]:
+def _sort_by_distance(qrs: Iterable[search_model.QueryResult[T]]) -> list[search_model.QueryResult[T]]:
     return sorted(qrs, key=lambda qr: qr.distance)
 
 
-def _query_results_to_temes(con: Connection) -> Callable[[Iterable[search_model.QueryResult]], Iterator[model.Tema]]:
+def _query_results_to_temes(
-    def fetch_temes(qrs: Iterable[search_model.QueryResult]) -> Iterator[model.Tema]:
+    con: Connection
-        return filter(None, map(lambda qr: temes_q.get_tema_by_id(tema_id=qr.id, con=con), qrs))
+) -> Callable[[Iterable[search_model.QueryResult[int]]], Iterator[model.Tema]]:
+    def fetch_temes(qrs: Iterable[search_model.QueryResult[int]]) -> Iterator[model.Tema]:
+        return filter(None, map(lambda qr: temes_q.get_tema_by_id(tema_id=qr.result, con=con), qrs))
     return fetch_temes
 
 
@@ -76,13 +82,35 @@ def _filter_hidden(hidden: bool) -> Callable[[Iterable[model.Tema]], Iterator[mo
     return filter_hidden
 
 
-def _apply_limit_offset(limit: int, offset: int) -> Callable[[Iterable[model.Tema]], list[model.Tema]]:
+def _filter_properties(properties: list[str]) -> Callable[[Iterable[model.Tema]], Iterator[model.Tema]]:
-    def apply_limit_offset(temes: Iterable[model.Tema]) -> list[model.Tema]:
+    properties_set = set(prop.lower() for prop in properties)
+
+    def has_properties(tema: model.Tema) -> bool:
+        tema_properties = {prop.value.lower() for prop in tema.properties}
+        return all(prop in tema_properties for prop in properties_set)
+
+    def filter_properties(temes: Iterable[model.Tema]) -> Iterator[model.Tema]:
+        return filter(has_properties, temes)
+
+    return filter_properties
+
+
+def _apply_limit_offset(limit: int, offset: int) -> Callable[[Iterable[T]], list[T]]:
+    def apply_limit_offset(temes: Iterable[T]) -> list[T]:
         return list(temes)[offset:offset + limit]
     return apply_limit_offset
 
 
-def busca_temes(query: str, hidden: bool = False, limit: int = 10, offset: int = 0) -> list[model.Tema]:
+def busca_temes(
+    query: str,
+    properties: list[str],
+    hidden: bool = False,
+    limit: int = 10,
+    offset: int = 0,
+) -> list[model.Tema]:
+    """
+    This function adds properties to Tema
+    """
     t0 = time.time()
     with get_connection() as con:
         result = (
@@ -92,7 +120,34 @@ def busca_temes(query: str, hidden: bool = False, limit: int = 10, offset: int =
             _sort_by_distance |
             _query_results_to_temes(con) |
             _filter_hidden(hidden) |
+            properties_service.add_properties_to_temes |
+            _filter_properties(properties) |
             _apply_limit_offset(limit=limit, offset=offset)
         ).result()
-    logger.info(f"Search time: { int((time.time() - t0) * 1000) } ms")
+    logger.info(f"Temes search time: { int((time.time() - t0) * 1000) } ms")
+    return result
+
+
+def _extract_properties(query_results: list[search_model.QueryResult[str]]) -> list[str]:
+    return [qr.result for qr in query_results]
+
+
+def busca_properties(
+    query: str,
+    limit: int = 10,
+    offset: int = 0,
+) -> list[str]:
+    if not query:
+        return []
+    t0 = time.time()
+    with get_connection() as con:
+        result = (
+            FnChain.transform(properties_dal.get_property_value_to_ngrams(con).items()) |
+            _build_results_fn(query) |
+            _filter_distance |
+            _sort_by_distance |
+            _apply_limit_offset(limit=limit, offset=offset) |
+            _extract_properties
+        ).result()
+    logger.info(f"Properties search time: { int((time.time() - t0) * 1000) } ms")
     return result