From c0df3440f59ab8346dd30805ddd3682cb2163e9b Mon Sep 17 00:00:00 2001 From: shibao Date: Sun, 27 Nov 2022 21:10:54 -0500 Subject: [PATCH] improve search --- changelog.md | 5 ++ lib/memex/contexts.ex | 8 +- lib/memex/notes.ex | 8 +- lib/memex/pipelines.ex | 8 +- mix.exs | 2 +- .../migrations/20221128003712_fix_search.exs | 56 ++++++++++++++ test/memex/contexts_test.exs | 75 ++++++++++++++++++ test/memex/notes_test.exs | 77 +++++++++++++++++++ test/memex/pipelines_test.exs | 75 ++++++++++++++++++ test/support/fixtures/contexts_fixtures.ex | 2 +- test/support/fixtures/notes_fixtures.ex | 2 +- test/support/fixtures/pipelines_fixtures.ex | 2 +- 12 files changed, 304 insertions(+), 16 deletions(-) create mode 100644 changelog.md create mode 100644 priv/repo/migrations/20221128003712_fix_search.exs diff --git a/changelog.md b/changelog.md new file mode 100644 index 0000000..c4976dd --- /dev/null +++ b/changelog.md @@ -0,0 +1,5 @@ +# v0.1.1 +- improve search a whole lot + +# v0.1.0 +initial release >:3c diff --git a/lib/memex/contexts.ex b/lib/memex/contexts.ex index fdf1062..8543b25 100644 --- a/lib/memex/contexts.ex +++ b/lib/memex/contexts.ex @@ -35,13 +35,13 @@ defmodule Memex.Contexts do where: c.user_id == ^user_id, where: fragment( - "search @@ to_tsquery(websearch_to_tsquery(?)::text || ':*')", + "search @@ websearch_to_tsquery('english', ?)", ^trimmed_search ), order_by: { :desc, fragment( - "ts_rank_cd(search, to_tsquery(websearch_to_tsquery(?)::text || ':*'), 4)", + "ts_rank_cd(search, websearch_to_tsquery('english', ?), 4)", ^trimmed_search ) } @@ -76,13 +76,13 @@ defmodule Memex.Contexts do where: c.visibility == :public, where: fragment( - "search @@ to_tsquery(websearch_to_tsquery(?)::text || ':*')", + "search @@ websearch_to_tsquery('english', ?)", ^trimmed_search ), order_by: { :desc, fragment( - "ts_rank_cd(search, to_tsquery(websearch_to_tsquery(?)::text || ':*'), 4)", + "ts_rank_cd(search, websearch_to_tsquery('english', ?), 4)", ^trimmed_search ) } diff --git a/lib/memex/notes.ex b/lib/memex/notes.ex index 34b2ea5..b86aca8 100644 --- a/lib/memex/notes.ex +++ b/lib/memex/notes.ex @@ -35,13 +35,13 @@ defmodule Memex.Notes do where: n.user_id == ^user_id, where: fragment( - "search @@ to_tsquery(websearch_to_tsquery(?)::text || ':*')", + "search @@ websearch_to_tsquery('english', ?)", ^trimmed_search ), order_by: { :desc, fragment( - "ts_rank_cd(search, to_tsquery(websearch_to_tsquery(?)::text || ':*'), 4)", + "ts_rank_cd(search, websearch_to_tsquery('english', ?), 4)", ^trimmed_search ) } @@ -75,13 +75,13 @@ defmodule Memex.Notes do where: n.visibility == :public, where: fragment( - "search @@ to_tsquery(websearch_to_tsquery(?)::text || ':*')", + "search @@ websearch_to_tsquery('english', ?)", ^trimmed_search ), order_by: { :desc, fragment( - "ts_rank_cd(search, to_tsquery(websearch_to_tsquery(?)::text || ':*'), 4)", + "ts_rank_cd(search, websearch_to_tsquery('english', ?), 4)", ^trimmed_search ) } diff --git a/lib/memex/pipelines.ex b/lib/memex/pipelines.ex index 09384d7..f72a1a1 100644 --- a/lib/memex/pipelines.ex +++ b/lib/memex/pipelines.ex @@ -35,13 +35,13 @@ defmodule Memex.Pipelines do where: p.user_id == ^user_id, where: fragment( - "search @@ to_tsquery(websearch_to_tsquery(?)::text || ':*')", + "search @@ websearch_to_tsquery('english', ?)", ^trimmed_search ), order_by: { :desc, fragment( - "ts_rank_cd(search, to_tsquery(websearch_to_tsquery(?)::text || ':*'), 4)", + "ts_rank_cd(search, websearch_to_tsquery('english', ?), 4)", ^trimmed_search ) } @@ -75,13 +75,13 @@ defmodule Memex.Pipelines do where: p.visibility == :public, where: fragment( - "search @@ to_tsquery(websearch_to_tsquery(?)::text || ':*')", + "search @@ websearch_to_tsquery('english', ?)", ^trimmed_search ), order_by: { :desc, fragment( - "ts_rank_cd(search, to_tsquery(websearch_to_tsquery(?)::text || ':*'), 4)", + "ts_rank_cd(search, websearch_to_tsquery('english', ?), 4)", ^trimmed_search ) } diff --git a/mix.exs b/mix.exs index 4169209..ae757e6 100644 --- a/mix.exs +++ b/mix.exs @@ -4,7 +4,7 @@ defmodule Memex.MixProject do def project do [ app: :memex, - version: "0.1.0", + version: "0.1.1", elixir: "~> 1.14", elixirc_paths: elixirc_paths(Mix.env()), compilers: Mix.compilers(), diff --git a/priv/repo/migrations/20221128003712_fix_search.exs b/priv/repo/migrations/20221128003712_fix_search.exs new file mode 100644 index 0000000..b30a13f --- /dev/null +++ b/priv/repo/migrations/20221128003712_fix_search.exs @@ -0,0 +1,56 @@ +defmodule Memex.Repo.Migrations.FixSearch do + use Ecto.Migration + + def up do + reset_search_columns() + end + + def down do + # no way to rollback this migration since the previous generated search columns were invalid + reset_search_columns() + end + + defp reset_search_columns() do + alter table(:notes), do: remove(:search) + alter table(:contexts), do: remove(:search) + alter table(:pipelines), do: remove(:search) + + flush() + + execute """ + ALTER TABLE notes + ADD COLUMN search tsvector + GENERATED ALWAYS AS ( + setweight(to_tsvector('english', coalesce(slug, '')), 'A') || + setweight(to_tsvector('english', coalesce(immutable_array_to_string(tags, ' '), '')), 'B') || + setweight(to_tsvector('english', coalesce(content, '')), 'C') + ) STORED + """ + + execute("CREATE INDEX notes_trgm_idx ON notes USING GIN (search)") + + execute """ + ALTER TABLE contexts + ADD COLUMN search tsvector + GENERATED ALWAYS AS ( + setweight(to_tsvector('english', coalesce(slug, '')), 'A') || + setweight(to_tsvector('english', coalesce(immutable_array_to_string(tags, ' '), '')), 'B') || + setweight(to_tsvector('english', coalesce(content, '')), 'C') + ) STORED + """ + + execute("CREATE INDEX contexts_trgm_idx ON contexts USING GIN (search)") + + execute """ + ALTER TABLE pipelines + ADD COLUMN search tsvector + GENERATED ALWAYS AS ( + setweight(to_tsvector('english', coalesce(slug, '')), 'A') || + setweight(to_tsvector('english', coalesce(immutable_array_to_string(tags, ' '), '')), 'B') || + setweight(to_tsvector('english', coalesce(description, '')), 'C') + ) STORED + """ + + execute("CREATE INDEX pipelines_trgm_idx ON pipelines USING GIN (search)") + end +end diff --git a/test/memex/contexts_test.exs b/test/memex/contexts_test.exs index c4da056..c80961f 100644 --- a/test/memex/contexts_test.exs +++ b/test/memex/contexts_test.exs @@ -17,6 +17,36 @@ defmodule Memex.ContextsTest do assert Contexts.list_contexts(user) == [context_a, context_b, context_c] end + test "list_contexts/2 returns relevant contexts for a user", %{user: user} do + context_a = context_fixture(%{slug: "dogs", content: "has some treats in it"}, user) + context_b = context_fixture(%{slug: "cats", tags: ["home"]}, user) + + context_c = + %{slug: "chickens", content: "bananas stuff", tags: ["life", "decisions"]} + |> context_fixture(user) + + _shouldnt_return = + %{slug: "dog", content: "banana treat stuff", visibility: :private} + |> context_fixture(user_fixture()) + + # slug + assert Contexts.list_contexts("dog", user) == [context_a] + assert Contexts.list_contexts("dogs", user) == [context_a] + assert Contexts.list_contexts("cat", user) == [context_b] + assert Contexts.list_contexts("chicken", user) == [context_c] + + # content + assert Contexts.list_contexts("treat", user) == [context_a] + assert Contexts.list_contexts("banana", user) == [context_c] + assert Contexts.list_contexts("stuff", user) == [context_c] + + # tag + assert Contexts.list_contexts("home", user) == [context_b] + assert Contexts.list_contexts("life", user) == [context_c] + assert Contexts.list_contexts("decision", user) == [context_c] + assert Contexts.list_contexts("decisions", user) == [context_c] + end + test "list_public_contexts/0 returns public contexts", %{user: user} do public_context = context_fixture(%{visibility: :public}, user) context_fixture(%{visibility: :unlisted}, user) @@ -24,6 +54,51 @@ defmodule Memex.ContextsTest do assert Contexts.list_public_contexts() == [public_context] end + test "list_public_contexts/1 returns relevant contexts for a user", %{user: user} do + context_a = + %{slug: "dogs", content: "has some treats in it", visibility: :public} + |> context_fixture(user) + + context_b = + %{slug: "cats", tags: ["home"], visibility: :public} + |> context_fixture(user) + + context_c = + %{ + slug: "chickens", + content: "bananas stuff", + tags: ["life", "decisions"], + visibility: :public + } + |> context_fixture(user) + + _shouldnt_return = + %{ + slug: "dog", + content: "treats bananas stuff", + tags: ["home", "life", "decisions"], + visibility: :private + } + |> context_fixture(user) + + # slug + assert Contexts.list_public_contexts("dog") == [context_a] + assert Contexts.list_public_contexts("dogs") == [context_a] + assert Contexts.list_public_contexts("cat") == [context_b] + assert Contexts.list_public_contexts("chicken") == [context_c] + + # content + assert Contexts.list_public_contexts("treat") == [context_a] + assert Contexts.list_public_contexts("banana") == [context_c] + assert Contexts.list_public_contexts("stuff") == [context_c] + + # tag + assert Contexts.list_public_contexts("home") == [context_b] + assert Contexts.list_public_contexts("life") == [context_c] + assert Contexts.list_public_contexts("decision") == [context_c] + assert Contexts.list_public_contexts("decisions") == [context_c] + end + test "get_context!/1 returns the context with given id", %{user: user} do context = context_fixture(%{visibility: :public}, user) assert Contexts.get_context!(context.id, user) == context diff --git a/test/memex/notes_test.exs b/test/memex/notes_test.exs index 788505b..e49ba90 100644 --- a/test/memex/notes_test.exs +++ b/test/memex/notes_test.exs @@ -14,9 +14,41 @@ defmodule Memex.NotesTest do note_a = note_fixture(%{slug: "a", visibility: :public}, user) note_b = note_fixture(%{slug: "b", visibility: :unlisted}, user) note_c = note_fixture(%{slug: "c", visibility: :private}, user) + _shouldnt_return = note_fixture(%{visibility: :private}, user_fixture()) + assert Notes.list_notes(user) == [note_a, note_b, note_c] end + test "list_notes/2 returns relevant notes for a user", %{user: user} do + note_a = note_fixture(%{slug: "dogs", content: "has some treats in it"}, user) + note_b = note_fixture(%{slug: "cats", tags: ["home"]}, user) + + note_c = + %{slug: "chickens", content: "bananas stuff", tags: ["life", "decisions"]} + |> note_fixture(user) + + _shouldnt_return = + %{slug: "dog", content: "banana treat stuff", visibility: :private} + |> note_fixture(user_fixture()) + + # slug + assert Notes.list_notes("dog", user) == [note_a] + assert Notes.list_notes("dogs", user) == [note_a] + assert Notes.list_notes("cat", user) == [note_b] + assert Notes.list_notes("chicken", user) == [note_c] + + # content + assert Notes.list_notes("treat", user) == [note_a] + assert Notes.list_notes("banana", user) == [note_c] + assert Notes.list_notes("stuff", user) == [note_c] + + # tag + assert Notes.list_notes("home", user) == [note_b] + assert Notes.list_notes("life", user) == [note_c] + assert Notes.list_notes("decision", user) == [note_c] + assert Notes.list_notes("decisions", user) == [note_c] + end + test "list_public_notes/0 returns public notes", %{user: user} do public_note = note_fixture(%{visibility: :public}, user) note_fixture(%{visibility: :unlisted}, user) @@ -24,6 +56,51 @@ defmodule Memex.NotesTest do assert Notes.list_public_notes() == [public_note] end + test "list_public_notes/1 returns relevant notes for a user", %{user: user} do + note_a = + %{slug: "dogs", content: "has some treats in it", visibility: :public} + |> note_fixture(user) + + note_b = + %{slug: "cats", tags: ["home"], visibility: :public} + |> note_fixture(user) + + note_c = + %{ + slug: "chickens", + content: "bananas stuff", + tags: ["life", "decisions"], + visibility: :public + } + |> note_fixture(user) + + _shouldnt_return = + %{ + slug: "dog", + content: "treats bananas stuff", + tags: ["home", "life", "decisions"], + visibility: :private + } + |> note_fixture(user) + + # slug + assert Notes.list_public_notes("dog") == [note_a] + assert Notes.list_public_notes("dogs") == [note_a] + assert Notes.list_public_notes("cat") == [note_b] + assert Notes.list_public_notes("chicken") == [note_c] + + # content + assert Notes.list_public_notes("treat") == [note_a] + assert Notes.list_public_notes("banana") == [note_c] + assert Notes.list_public_notes("stuff") == [note_c] + + # tag + assert Notes.list_public_notes("home") == [note_b] + assert Notes.list_public_notes("life") == [note_c] + assert Notes.list_public_notes("decision") == [note_c] + assert Notes.list_public_notes("decisions") == [note_c] + end + test "get_note!/1 returns the note with given id", %{user: user} do note = note_fixture(%{visibility: :public}, user) assert Notes.get_note!(note.id, user) == note diff --git a/test/memex/pipelines_test.exs b/test/memex/pipelines_test.exs index 074a553..45f3712 100644 --- a/test/memex/pipelines_test.exs +++ b/test/memex/pipelines_test.exs @@ -17,6 +17,36 @@ defmodule Memex.PipelinesTest do assert Pipelines.list_pipelines(user) == [pipeline_a, pipeline_b, pipeline_c] end + test "list_pipelines/2 returns relevant pipelines for a user", %{user: user} do + pipeline_a = pipeline_fixture(%{slug: "dogs", description: "has some treats in it"}, user) + pipeline_b = pipeline_fixture(%{slug: "cats", tags: ["home"]}, user) + + pipeline_c = + %{slug: "chickens", description: "bananas stuff", tags: ["life", "decisions"]} + |> pipeline_fixture(user) + + _shouldnt_return = + %{slug: "dog", description: "banana treat stuff", visibility: :private} + |> pipeline_fixture(user_fixture()) + + # slug + assert Pipelines.list_pipelines("dog", user) == [pipeline_a] + assert Pipelines.list_pipelines("dogs", user) == [pipeline_a] + assert Pipelines.list_pipelines("cat", user) == [pipeline_b] + assert Pipelines.list_pipelines("chicken", user) == [pipeline_c] + + # description + assert Pipelines.list_pipelines("treat", user) == [pipeline_a] + assert Pipelines.list_pipelines("banana", user) == [pipeline_c] + assert Pipelines.list_pipelines("stuff", user) == [pipeline_c] + + # tag + assert Pipelines.list_pipelines("home", user) == [pipeline_b] + assert Pipelines.list_pipelines("life", user) == [pipeline_c] + assert Pipelines.list_pipelines("decision", user) == [pipeline_c] + assert Pipelines.list_pipelines("decisions", user) == [pipeline_c] + end + test "list_public_pipelines/0 returns public pipelines", %{user: user} do public_pipeline = pipeline_fixture(%{visibility: :public}, user) pipeline_fixture(%{visibility: :unlisted}, user) @@ -24,6 +54,51 @@ defmodule Memex.PipelinesTest do assert Pipelines.list_public_pipelines() == [public_pipeline] end + test "list_public_pipelines/1 returns relevant pipelines for a user", %{user: user} do + pipeline_a = + %{slug: "dogs", description: "has some treats in it", visibility: :public} + |> pipeline_fixture(user) + + pipeline_b = + %{slug: "cats", tags: ["home"], visibility: :public} + |> pipeline_fixture(user) + + pipeline_c = + %{ + slug: "chickens", + description: "bananas stuff", + tags: ["life", "decisions"], + visibility: :public + } + |> pipeline_fixture(user) + + _shouldnt_return = + %{ + slug: "dog", + description: "treats bananas stuff", + tags: ["home", "life", "decisions"], + visibility: :private + } + |> pipeline_fixture(user) + + # slug + assert Pipelines.list_public_pipelines("dog") == [pipeline_a] + assert Pipelines.list_public_pipelines("dogs") == [pipeline_a] + assert Pipelines.list_public_pipelines("cat") == [pipeline_b] + assert Pipelines.list_public_pipelines("chicken") == [pipeline_c] + + # description + assert Pipelines.list_public_pipelines("treat") == [pipeline_a] + assert Pipelines.list_public_pipelines("banana") == [pipeline_c] + assert Pipelines.list_public_pipelines("stuff") == [pipeline_c] + + # tag + assert Pipelines.list_public_pipelines("home") == [pipeline_b] + assert Pipelines.list_public_pipelines("life") == [pipeline_c] + assert Pipelines.list_public_pipelines("decision") == [pipeline_c] + assert Pipelines.list_public_pipelines("decisions") == [pipeline_c] + end + test "get_pipeline!/1 returns the pipeline with given id", %{user: user} do pipeline = pipeline_fixture(%{visibility: :public}, user) assert Pipelines.get_pipeline!(pipeline.id, user) == pipeline diff --git a/test/support/fixtures/contexts_fixtures.ex b/test/support/fixtures/contexts_fixtures.ex index f034543..56e0d78 100644 --- a/test/support/fixtures/contexts_fixtures.ex +++ b/test/support/fixtures/contexts_fixtures.ex @@ -16,7 +16,7 @@ defmodule Memex.ContextsFixtures do attrs |> Enum.into(%{ content: "some content", - tag: [], + tags: [], slug: random_slug(), visibility: :private }) diff --git a/test/support/fixtures/notes_fixtures.ex b/test/support/fixtures/notes_fixtures.ex index 2c593e6..1d92190 100644 --- a/test/support/fixtures/notes_fixtures.ex +++ b/test/support/fixtures/notes_fixtures.ex @@ -16,7 +16,7 @@ defmodule Memex.NotesFixtures do attrs |> Enum.into(%{ content: "some content", - tag: [], + tags: [], slug: random_slug(), visibility: :private }) diff --git a/test/support/fixtures/pipelines_fixtures.ex b/test/support/fixtures/pipelines_fixtures.ex index 2cd6069..a189669 100644 --- a/test/support/fixtures/pipelines_fixtures.ex +++ b/test/support/fixtures/pipelines_fixtures.ex @@ -16,7 +16,7 @@ defmodule Memex.PipelinesFixtures do attrs |> Enum.into(%{ description: "some description", - tag: [], + tags: [], slug: random_slug(), visibility: :private })