From 0d13f602fe1889282300a248a8dc0ce03b09b8e4 Mon Sep 17 00:00:00 2001 From: Brian Pepple Date: Tue, 3 Nov 2026 11:38:10 -0500 Subject: [PATCH 1/4] Search creators by surname when the query starts with a space Editors often search for a credit's creator by typing a space and then the surname, e.g. " Cameron". The old autocomplete passed the space through to icontains, so first names were skipped. izaya stripped it, so " Cameron" matched "Cameron Stewart" and even ranked first names above surnames. Use izaya 0.4.0's clean_query hook to keep one leading space in CreatorLookup. With a leading space, search matches only names and aliases where the text follows a space, and puts names that end with it first. Searches without a leading space are unchanged. --- comicsdb/lookups.py | 29 +++++++++++++++- pyproject.toml | 2 +- tests/comicsdb/test_lookups.py | 62 ++++++++++++++++++++++++++++++++++ uv.lock | 8 ++--- 4 files changed, 95 insertions(+), 6 deletions(-) diff --git a/comicsdb/lookups.py b/comicsdb/lookups.py index 2fa4fa15..231a6330 100644 --- a/comicsdb/lookups.py +++ b/comicsdb/lookups.py @@ -4,7 +4,7 @@ import re from functools import reduce -from django.db.models import Q, QuerySet +from django.db.models import Case, Q, QuerySet, Value, When from izaya import ModelLookup, register from comicsdb.models import Arc, Character, Creator, Publisher, Series, Team, Universe @@ -55,6 +55,33 @@ class CreatorLookup(ModelLookup): search_fields = ["name__unaccent__icontains", "alias__icontains"] rank_fields = ["name__unaccent"] + def clean_query(self, query: str) -> str: + """Keep one leading space, which editors type to search by surname (e.g. " Cameron").""" + text = query.strip() + return f" {text}" if text and query[0].isspace() else text + + def search(self, query, request): + """ + Search creators, treating a leading space as a search for a later name. + + " Cameron" only matches names with "Cameron" after a space, so "Don Cameron" matches but + "Cameron Stewart" doesn't. Names ending with the search text come first. + """ + if not query.startswith(" "): + return super().search(query, request) + + queryset = ( + self.get_queryset(request) + .filter(Q(name__unaccent__icontains=query) | Q(alias__icontains=query)) + .alias( + surname_rank=Case( + When(name__unaccent__iendswith=query, then=Value(0)), default=Value(1) + ) + ) + .order_by("surname_rank", *self.model._meta.ordering) + ) + return self.rank(queryset, query.strip()) + @register class RoleLookup(ModelLookup): diff --git a/pyproject.toml b/pyproject.toml index 5943a4ca..33c857f5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,7 @@ dependencies = [ "gunicorn", "imagehash", "isbnlib", - "izaya>=0.3.0", + "izaya>=0.4.0", "markdown", "pillow", "psycopg[binary,pool]", diff --git a/tests/comicsdb/test_lookups.py b/tests/comicsdb/test_lookups.py index b784316c..dcb5b9fc 100644 --- a/tests/comicsdb/test_lookups.py +++ b/tests/comicsdb/test_lookups.py @@ -4,6 +4,7 @@ import pytest from django.urls import reverse +from django.utils.text import slugify from izaya.lookups import registry from comicsdb.lookups import ( @@ -621,6 +622,67 @@ def test_search_no_match(self, john_byrne, walter_simonson): assert queryset.count() == 0 + @pytest.fixture + def camerons(self, create_user): + """Create creators with "Cameron" as a first, middle and last name.""" + user = create_user() + return { + name: Creator.objects.create( + name=name, slug=slugify(name), edited_by=user, created_by=user + ) + for name in ("Cameron Stewart", "Don Cameron", "Jo Cameron Smith", "Ann Camerons") + } + + def test_leading_space_is_kept(self): + """Test that one leading space survives cleaning, so it can mean a surname search.""" + assert CreatorLookup().clean_query(" Cameron ") == " Cameron" + + def test_surrounding_whitespace_is_stripped_without_a_leading_space(self): + """Test that a query without a leading space is stripped as usual.""" + assert CreatorLookup().clean_query("Cameron ") == "Cameron" + + def test_leading_space_skips_first_names(self, camerons): + """Test that a leading space only matches names after the first.""" + names = list(CreatorLookup().search(" cameron", None).values_list("name", flat=True)) + + assert "Cameron Stewart" not in names + assert set(names) == {"Don Cameron", "Jo Cameron Smith", "Ann Camerons"} + + def test_leading_space_puts_names_ending_with_the_query_first(self, camerons): + """Test that surname matches come before middle names and partial words.""" + names = list(CreatorLookup().search(" cameron", None).values_list("name", flat=True)) + + assert names[0] == "Don Cameron" + + def test_leading_space_matches_aliases(self, create_user): + """Test that a leading space also searches later names in aliases.""" + user = create_user() + creator = Creator.objects.create( + name="Frank Miller", + slug="frank-miller", + alias=["Frank Miller Sr."], + edited_by=user, + created_by=user, + ) + + assert creator in CreatorLookup().search(" sr.", None) + + def test_without_leading_space_first_names_match(self, camerons): + """Test that a plain query still matches first names, ranked first.""" + names = list(CreatorLookup().search("cameron", None).values_list("name", flat=True)) + + assert names[0] == "Cameron Stewart" + assert len(names) == len(camerons) + + def test_search_endpoint_keeps_leading_space(self, client, create_user, camerons): + """Test that the search endpoint passes the leading space through to the lookup.""" + client.force_login(create_user()) + resp = client.get(reverse("izaya:search", args=["CreatorLookup"]), {"q": " Cameron"}) + + assert resp.status_code == HTTP_200_OK + assert b"Don Cameron" in resp.content + assert b"Cameron Stewart" not in resp.content + def test_get_label_for_record(self, john_byrne): """Test that get_label_for_record returns string representation.""" label = CreatorLookup().label(john_byrne) diff --git a/uv.lock b/uv.lock index 7bdf527c..3caf567f 100644 --- a/uv.lock +++ b/uv.lock @@ -604,14 +604,14 @@ wheels = [ [[package]] name = "izaya" -version = "0.3.0" +version = "0.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "django" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/34/a5/3dd8d822e93aa60c95be587d9b9bed1dbd2bb209b63ed890e9f3f8d10ae3/izaya-0.3.0.tar.gz", hash = "sha256:9dc27cbe42d3a66c0009cad83a5cc56d077af97fd9003a02c1855bd32eae540a", size = 30770, upload-time = "2026-11-01T19:45:04.45Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e8/a7/e7842bf8d8a090c764d6cc71438171b13e6eee6da46c83ee8af696f834ad/izaya-0.4.0.tar.gz", hash = "sha256:d56f9170d7d52743f91766f01ca6f8afd7219b360c0f46ee0fc0b6d4fe05767d", size = 26929, upload-time = "2026-11-03T16:26:41.427Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/54/1b/e8920af7b1b1d0cb59003180154c8b37155d3a218f64f055fac010e20289/izaya-0.3.0-py3-none-any.whl", hash = "sha256:a12602812507e1a4332681e0d39723bf80307272053ef93cbf0427875282776e", size = 29797, upload-time = "2026-11-01T19:45:03.458Z" }, + { url = "https://files.pythonhosted.org/packages/3a/2b/812a7cb486f0e89ae0dafab7784020d43c8badf000f34f07087ea8c7b576/izaya-0.4.0-py3-none-any.whl", hash = "sha256:199c363c9b7e0ab6a737b10c1913e39dbfa302a74c8ee536051ee153b50d3d4e", size = 27616, upload-time = "2026-11-03T16:26:40.442Z" }, ] [[package]] @@ -766,7 +766,7 @@ requires-dist = [ { name = "gunicorn" }, { name = "imagehash" }, { name = "isbnlib" }, - { name = "izaya", specifier = ">=0.3.0" }, + { name = "izaya", specifier = ">=0.4.0" }, { name = "markdown" }, { name = "pillow" }, { name = "psycopg", extras = ["binary", "pool"] }, From 1bc8552773a494e80d1d865b88fc41aa08536c7a Mon Sep 17 00:00:00 2001 From: Brian Pepple Date: Tue, 3 Nov 2026 11:47:53 -0500 Subject: [PATCH 2/4] Fix ordering and edge cases in creator surname search - Sort leading-space results by surname match instead of the general rank, which put names starting with the query (found through an alias) ahead of the surname matches - Drop the leading space when fewer than min_length characters follow it, so " Ca" shows the minimum-length message instead of searching - Only treat a plain space as the surname prefix, and accept several leading spaces in search() - Build the filter from search_fields and keep the distinct() guard --- comicsdb/lookups.py | 34 +++++++++++++++++++++------------- tests/comicsdb/test_lookups.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 13 deletions(-) diff --git a/comicsdb/lookups.py b/comicsdb/lookups.py index 231a6330..850f6c3c 100644 --- a/comicsdb/lookups.py +++ b/comicsdb/lookups.py @@ -56,9 +56,16 @@ class CreatorLookup(ModelLookup): rank_fields = ["name__unaccent"] def clean_query(self, query: str) -> str: - """Keep one leading space, which editors type to search by surname (e.g. " Cameron").""" + """Keep one leading space, which editors type to search by surname (e.g. " Cameron"). + + Only a plain space counts, so pasted text with a leading tab or NBSP searches normally. + The space is dropped when too little text follows it, so the view's ``min_length`` check + counts only the characters actually typed. + """ text = query.strip() - return f" {text}" if text and query[0].isspace() else text + if query.startswith(" ") and len(text) >= self.min_length: + return f" {text}" + return text def search(self, query, request): """ @@ -70,17 +77,18 @@ def search(self, query, request): if not query.startswith(" "): return super().search(query, request) - queryset = ( - self.get_queryset(request) - .filter(Q(name__unaccent__icontains=query) | Q(alias__icontains=query)) - .alias( - surname_rank=Case( - When(name__unaccent__iendswith=query, then=Value(0)), default=Value(1) - ) - ) - .order_by("surname_rank", *self.model._meta.ordering) - ) - return self.rank(queryset, query.strip()) + query = f" {query.lstrip()}" + conditions = [Q(**{field: query}) for field in self.search_fields] + queryset = self.get_queryset(request).filter(reduce(operator.or_, conditions)) + surname_rank = Case(When(name__unaccent__iendswith=query, then=Value(0)), default=Value(1)) + if self.spawns_duplicates(self.search_fields): + # SELECT DISTINCT needs the ordering expression in the select list. + queryset = queryset.distinct().annotate(surname_rank=surname_rank) + else: + queryset = queryset.alias(surname_rank=surname_rank) + # rank() isn't used here: it would rank names starting with the stripped query (matched + # only through an alias) ahead of the surname matches. + return queryset.order_by("surname_rank", *self.model._meta.ordering, "pk") @register diff --git a/tests/comicsdb/test_lookups.py b/tests/comicsdb/test_lookups.py index dcb5b9fc..70cebf6a 100644 --- a/tests/comicsdb/test_lookups.py +++ b/tests/comicsdb/test_lookups.py @@ -654,6 +654,40 @@ def test_leading_space_puts_names_ending_with_the_query_first(self, camerons): assert names[0] == "Don Cameron" + def test_short_leading_space_query_drops_the_space(self): + """Test that the leading space doesn't count toward the minimum length.""" + assert CreatorLookup().clean_query(" Ca") == "Ca" + + def test_leading_non_space_whitespace_is_stripped(self): + """Test that a leading tab or NBSP doesn't trigger a surname search.""" + assert CreatorLookup().clean_query("\tCameron") == "Cameron" + assert CreatorLookup().clean_query("\u00a0Cameron") == "Cameron" + + def test_several_leading_spaces_search_by_surname(self, camerons): + """Test that search() treats several leading spaces like one.""" + names = list(CreatorLookup().search(" cameron", None).values_list("name", flat=True)) + + assert set(names) == {"Don Cameron", "Jo Cameron Smith", "Ann Camerons"} + + def test_leading_space_alias_match_does_not_outrank_surnames(self, camerons): + """Test that a first-name match found through an alias sorts after surname matches.""" + creator = camerons["Cameron Stewart"] + creator.alias = ["John Cameron Jr."] + creator.save() + + names = list(CreatorLookup().search(" cameron", None).values_list("name", flat=True)) + + assert names[0] == "Don Cameron" + assert names.index("Cameron Stewart") > names.index("Don Cameron") + + def test_short_leading_space_query_shows_min_length_message(self, client, create_user): + """Test that the endpoint asks for more text when only two characters follow the space.""" + client.force_login(create_user()) + resp = client.get(reverse("izaya:search", args=["CreatorLookup"]), {"q": " Ca"}) + + assert resp.status_code == HTTP_200_OK + assert str(CreatorLookup.min_length) in resp.content.decode() + def test_leading_space_matches_aliases(self, create_user): """Test that a leading space also searches later names in aliases.""" user = create_user() From 97f204c2f366f21576fcad5b20be558b1cf3448d Mon Sep 17 00:00:00 2001 From: Brian Pepple Date: Tue, 3 Nov 2026 11:51:48 -0500 Subject: [PATCH 3/4] Fix creator surname search and match accented queries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Sort leading-space results by surname match instead of the general rank, which put names starting with the query (found through an alias) ahead of the surname matches - Drop the leading space when fewer than min_length characters follow it, so " Ca" shows the minimum-length message instead of searching - Only treat a plain space as the surname prefix, and accept several leading spaces in search() - Unaccent the query with PostgreSQL's unaccent() for fields searched through __unaccent__, so "Peña" matches "Ana Peña" in every lookup --- comicsdb/lookups.py | 67 +++++++++++++++++++++++++++------- tests/comicsdb/test_lookups.py | 52 ++++++++++++++++++++++++++ 2 files changed, 105 insertions(+), 14 deletions(-) diff --git a/comicsdb/lookups.py b/comicsdb/lookups.py index 850f6c3c..147c5a35 100644 --- a/comicsdb/lookups.py +++ b/comicsdb/lookups.py @@ -4,6 +4,7 @@ import re from functools import reduce +from django.db import connection from django.db.models import Case, Q, QuerySet, Value, When from izaya import ModelLookup, register @@ -13,6 +14,43 @@ from comicsdb.models.issue import Issue +def unaccent(text: str) -> str: + """Return ``text`` with accents removed by PostgreSQL's ``unaccent()``. + + Searches compare ``unaccent(field)`` with the query, so the query has to be unaccented by + the same rules too, or an accented query like "Peña" never matches. ASCII text is returned + as is, which saves a query for most searches. + """ + if text.isascii(): + return text + with connection.cursor() as cursor: + cursor.execute("SELECT unaccent(%s)", [text]) + return cursor.fetchone()[0] + + +class UnaccentModelLookup(ModelLookup): + """A ModelLookup that also strips accents from the query. + + ``search_fields`` that go through ``__unaccent__`` get the unaccented query, while others, + such as ArrayFields, get the query as typed. ``rank_fields`` must all be unaccented. + """ + + def filter_queryset(self, query: str, request) -> QuerySet: + """Return the records where any of ``search_fields`` match ``query``.""" + plain = unaccent(query) + conditions = [ + Q(**{field: plain if "__unaccent__" in field else query}) + for field in self.search_fields + ] + queryset = self.get_queryset(request).filter(reduce(operator.or_, conditions)) + if self.spawns_duplicates(self.search_fields): + queryset = queryset.distinct() + return queryset + + def search(self, query, request): + return self.rank(self.filter_queryset(query, request), unaccent(query)) + + class PublisherLabelMixin: """Add the publisher name to a lookup's labels, for models with a ``publisher`` FK.""" @@ -24,7 +62,7 @@ def label(self, record) -> str: @register -class ArcLookup(ModelLookup): +class ArcLookup(UnaccentModelLookup): """Lookup for searching story arcs.""" model = Arc @@ -33,7 +71,7 @@ class ArcLookup(ModelLookup): @register -class CharacterLookup(ModelLookup): +class CharacterLookup(UnaccentModelLookup): """Lookup for searching characters. Note: alias uses regular icontains as it's an ArrayField. @@ -45,7 +83,7 @@ class CharacterLookup(ModelLookup): @register -class CreatorLookup(ModelLookup): +class CreatorLookup(UnaccentModelLookup): """Lookup for searching creators. Note: alias uses regular icontains as it's an ArrayField. @@ -78,12 +116,13 @@ def search(self, query, request): return super().search(query, request) query = f" {query.lstrip()}" - conditions = [Q(**{field: query}) for field in self.search_fields] - queryset = self.get_queryset(request).filter(reduce(operator.or_, conditions)) - surname_rank = Case(When(name__unaccent__iendswith=query, then=Value(0)), default=Value(1)) - if self.spawns_duplicates(self.search_fields): + queryset = self.filter_queryset(query, request) + surname_rank = Case( + When(name__unaccent__iendswith=unaccent(query), then=Value(0)), default=Value(1) + ) + if queryset.query.distinct: # SELECT DISTINCT needs the ordering expression in the select list. - queryset = queryset.distinct().annotate(surname_rank=surname_rank) + queryset = queryset.annotate(surname_rank=surname_rank) else: queryset = queryset.alias(surname_rank=surname_rank) # rank() isn't used here: it would rank names starting with the stripped query (matched @@ -100,7 +139,7 @@ class RoleLookup(ModelLookup): @register -class PublisherLookup(ModelLookup): +class PublisherLookup(UnaccentModelLookup): """Lookup for searching publishers.""" model = Publisher @@ -109,7 +148,7 @@ class PublisherLookup(ModelLookup): @register -class SeriesLookup(PublisherLabelMixin, ModelLookup): +class SeriesLookup(PublisherLabelMixin, UnaccentModelLookup): """Lookup for searching series. Note: alt_names uses the `joined` transform so the search can hit the trigram index on @@ -126,7 +165,7 @@ def label(self, record: Series) -> str: @register -class TeamLookup(ModelLookup): +class TeamLookup(UnaccentModelLookup): """Lookup for searching teams.""" model = Team @@ -135,7 +174,7 @@ class TeamLookup(ModelLookup): @register -class UniverseLookup(PublisherLabelMixin, ModelLookup): +class UniverseLookup(PublisherLabelMixin, UnaccentModelLookup): """Lookup for searching universes.""" model = Universe @@ -144,7 +183,7 @@ class UniverseLookup(PublisherLabelMixin, ModelLookup): @register -class ImprintLookup(PublisherLabelMixin, ModelLookup): +class ImprintLookup(PublisherLabelMixin, UnaccentModelLookup): """Lookup for searching imprints.""" model = Imprint @@ -194,7 +233,7 @@ def search(self, query, request): # All series name words must match (if any remain after removing year) conditions.extend( - Q(series__name__unaccent__icontains=word) for word in series_part.split() + Q(series__name__unaccent__icontains=word) for word in unaccent(series_part).split() ) # Add number filter if there's text after '#' diff --git a/tests/comicsdb/test_lookups.py b/tests/comicsdb/test_lookups.py index 70cebf6a..a646a54b 100644 --- a/tests/comicsdb/test_lookups.py +++ b/tests/comicsdb/test_lookups.py @@ -17,6 +17,7 @@ SeriesLookup, TeamLookup, UniverseLookup, + unaccent, ) from comicsdb.models.arc import Arc from comicsdb.models.character import Character @@ -336,6 +337,12 @@ def test_search_with_series_and_nonexistent_number(self, all_test_issues): # Should return empty queryset assert queryset.count() == 0 + def test_search_with_pound_sign_and_accented_series(self, all_test_issues): + """Test that accents in the series part of a '#' search are ignored.""" + queryset = IssueLookup().search("Spíder#1", None) + + assert list(queryset) == [all_test_issues[0]] + def test_get_queryset_includes_series_info(self, spider_man_issue_1): """Test that get_queryset includes related series information.""" queryset = IssueLookup().get_base_queryset() @@ -608,6 +615,38 @@ def test_search_with_accent_in_name(self, create_user): assert creator.id in creator_ids + def test_search_with_accented_query(self, create_user): + """Test that an accented query matches, since the query is unaccented as well.""" + user = create_user() + creator = Creator.objects.create( + name="Ana Peña", slug="ana-pena", edited_by=user, created_by=user + ) + + assert creator in CreatorLookup().search("Peña", None) + + def test_leading_space_with_accented_query(self, create_user): + """Test that an accented surname search matches and gets surname priority.""" + user = create_user() + for name in ("Peña Ruiz", "Luis Peñaloza", "Ana Peña"): + Creator.objects.create(name=name, slug=slugify(name), edited_by=user, created_by=user) + + names = list(CreatorLookup().search(" Peña", None).values_list("name", flat=True)) + + assert names == ["Ana Peña", "Luis Peñaloza"] + + def test_accented_query_matches_accented_alias(self, create_user): + """Test that aliases, which aren't unaccented, still get the query as typed.""" + user = create_user() + creator = Creator.objects.create( + name="John Smith", + slug="john-smith", + alias=["Señor Smith"], + edited_by=user, + created_by=user, + ) + + assert creator in CreatorLookup().search("Señor", None) + def test_search_partial_match(self, john_byrne): """Test searching with partial name match.""" queryset = CreatorLookup().search("john", None) @@ -1107,6 +1146,19 @@ def test_selected_labels_use_one_query(self, lookup_case, django_assert_num_quer @pytest.mark.django_db +class TestUnaccent: + """Tests for the unaccent() helper.""" + + def test_uses_postgres_rules(self): + """Test that the query is unaccented by the same rules as the indexed field.""" + assert unaccent("Łódź Straße") == "Lodz Strasse" + + def test_ascii_text_skips_the_database(self, django_assert_num_queries): + """Test that plain ASCII text is returned without a query.""" + with django_assert_num_queries(0): + assert unaccent("Cameron") == "Cameron" + + class TestLookupRanking: """Tests for the relevance order of lookup results.""" From 9e834d8b5adb5d161c991acf4cc9535090117e62 Mon Sep 17 00:00:00 2001 From: Brian Pepple Date: Tue, 3 Nov 2026 11:55:48 -0500 Subject: [PATCH 4/4] Ignore accents in issue searches without '#' MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Search series names through the unaccented field and unaccent the query, matching the '#' path. "Spíder" now finds Spider-Man issues, and the search can use the series name trigram index. --- comicsdb/lookups.py | 8 ++++++-- tests/comicsdb/test_lookups.py | 6 ++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/comicsdb/lookups.py b/comicsdb/lookups.py index 147c5a35..7c3b6c04 100644 --- a/comicsdb/lookups.py +++ b/comicsdb/lookups.py @@ -192,11 +192,15 @@ class ImprintLookup(PublisherLabelMixin, UnaccentModelLookup): @register -class IssueLookup(ModelLookup): +class IssueLookup(UnaccentModelLookup): """Lookup for searching issues.""" model = Issue - search_fields = ["series__name__icontains", "number__icontains", "alt_number__icontains"] + search_fields = [ + "series__name__unaccent__icontains", + "number__icontains", + "alt_number__icontains", + ] def get_base_queryset(self) -> QuerySet: return super().get_base_queryset().select_related("series", "series__series_type") diff --git a/tests/comicsdb/test_lookups.py b/tests/comicsdb/test_lookups.py index a646a54b..c6571780 100644 --- a/tests/comicsdb/test_lookups.py +++ b/tests/comicsdb/test_lookups.py @@ -337,6 +337,12 @@ def test_search_with_series_and_nonexistent_number(self, all_test_issues): # Should return empty queryset assert queryset.count() == 0 + def test_search_without_pound_sign_and_accented_series(self, all_test_issues): + """Test that accents are ignored when searching series names without '#'.""" + queryset = IssueLookup().search("Spíder", None) + + assert set(queryset) == set(all_test_issues[:3]) + def test_search_with_pound_sign_and_accented_series(self, all_test_issues): """Test that accents in the series part of a '#' search are ignored.""" queryset = IssueLookup().search("Spíder#1", None)