improve search

changelog.md

@@ -0,0 +1,5 @@
+# v0.1.1
+- improve search a whole lot
+
+# v0.1.0
+initial release >:3c

lib/memex/contexts.ex

@@ -35,13 +35,13 @@ defmodule Memex.Contexts do
         where: c.user_id == ^user_id,
         where:
           fragment(
-            "search @@ to_tsquery(websearch_to_tsquery(?)::text || ':*')",
+            "search @@ websearch_to_tsquery('english', ?)",
             ^trimmed_search
           ),
         order_by: {
           :desc,
           fragment(
-            "ts_rank_cd(search, to_tsquery(websearch_to_tsquery(?)::text || ':*'), 4)",
+            "ts_rank_cd(search, websearch_to_tsquery('english', ?), 4)",
             ^trimmed_search
           )
         }
@@ -76,13 +76,13 @@ defmodule Memex.Contexts do
         where: c.visibility == :public,
         where:
           fragment(
-            "search @@ to_tsquery(websearch_to_tsquery(?)::text || ':*')",
+            "search @@ websearch_to_tsquery('english', ?)",
             ^trimmed_search
           ),
         order_by: {
           :desc,
           fragment(
-            "ts_rank_cd(search, to_tsquery(websearch_to_tsquery(?)::text || ':*'), 4)",
+            "ts_rank_cd(search, websearch_to_tsquery('english', ?), 4)",
             ^trimmed_search
           )
         }

lib/memex/notes.ex

@@ -35,13 +35,13 @@ defmodule Memex.Notes do
         where: n.user_id == ^user_id,
         where:
           fragment(
-            "search @@ to_tsquery(websearch_to_tsquery(?)::text || ':*')",
+            "search @@ websearch_to_tsquery('english', ?)",
             ^trimmed_search
           ),
         order_by: {
           :desc,
           fragment(
-            "ts_rank_cd(search, to_tsquery(websearch_to_tsquery(?)::text || ':*'), 4)",
+            "ts_rank_cd(search, websearch_to_tsquery('english', ?), 4)",
             ^trimmed_search
           )
         }
@@ -75,13 +75,13 @@ defmodule Memex.Notes do
         where: n.visibility == :public,
         where:
           fragment(
-            "search @@ to_tsquery(websearch_to_tsquery(?)::text || ':*')",
+            "search @@ websearch_to_tsquery('english', ?)",
             ^trimmed_search
           ),
         order_by: {
           :desc,
           fragment(
-            "ts_rank_cd(search, to_tsquery(websearch_to_tsquery(?)::text || ':*'), 4)",
+            "ts_rank_cd(search, websearch_to_tsquery('english', ?), 4)",
             ^trimmed_search
           )
         }

lib/memex/pipelines.ex

@@ -35,13 +35,13 @@ defmodule Memex.Pipelines do
         where: p.user_id == ^user_id,
         where:
           fragment(
-            "search @@ to_tsquery(websearch_to_tsquery(?)::text || ':*')",
+            "search @@ websearch_to_tsquery('english', ?)",
             ^trimmed_search
           ),
         order_by: {
           :desc,
           fragment(
-            "ts_rank_cd(search, to_tsquery(websearch_to_tsquery(?)::text || ':*'), 4)",
+            "ts_rank_cd(search, websearch_to_tsquery('english', ?), 4)",
             ^trimmed_search
           )
         }
@@ -75,13 +75,13 @@ defmodule Memex.Pipelines do
         where: p.visibility == :public,
         where:
           fragment(
-            "search @@ to_tsquery(websearch_to_tsquery(?)::text || ':*')",
+            "search @@ websearch_to_tsquery('english', ?)",
             ^trimmed_search
           ),
         order_by: {
           :desc,
           fragment(
-            "ts_rank_cd(search, to_tsquery(websearch_to_tsquery(?)::text || ':*'), 4)",
+            "ts_rank_cd(search, websearch_to_tsquery('english', ?), 4)",
             ^trimmed_search
           )
         }

mix.exs

@@ -4,7 +4,7 @@ defmodule Memex.MixProject do
   def project do
     [
       app: :memex,
-      version: "0.1.0",
+      version: "0.1.1",
       elixir: "~> 1.14",
       elixirc_paths: elixirc_paths(Mix.env()),
       compilers: Mix.compilers(),

priv/repo/migrations/20221128003712_fix_search.exs

@@ -0,0 +1,56 @@
+defmodule Memex.Repo.Migrations.FixSearch do
+  use Ecto.Migration
+
+  def up do
+    reset_search_columns()
+  end
+
+  def down do
+    # no way to rollback this migration since the previous generated search columns were invalid
+    reset_search_columns()
+  end
+
+  defp reset_search_columns() do
+    alter table(:notes), do: remove(:search)
+    alter table(:contexts), do: remove(:search)
+    alter table(:pipelines), do: remove(:search)
+
+    flush()
+
+    execute """
+    ALTER TABLE notes
+    ADD COLUMN search tsvector
+      GENERATED ALWAYS AS (
+        setweight(to_tsvector('english', coalesce(slug, '')), 'A') ||
+        setweight(to_tsvector('english', coalesce(immutable_array_to_string(tags, ' '), '')), 'B') ||
+        setweight(to_tsvector('english', coalesce(content, '')), 'C')
+      ) STORED
+    """
+
+    execute("CREATE INDEX notes_trgm_idx ON notes USING GIN (search)")
+
+    execute """
+    ALTER TABLE contexts
+      ADD COLUMN search tsvector
+      GENERATED ALWAYS AS (
+        setweight(to_tsvector('english', coalesce(slug, '')), 'A') ||
+        setweight(to_tsvector('english', coalesce(immutable_array_to_string(tags, ' '), '')), 'B') ||
+        setweight(to_tsvector('english', coalesce(content, '')), 'C')
+      ) STORED
+    """
+
+    execute("CREATE INDEX contexts_trgm_idx ON contexts USING GIN (search)")
+
+    execute """
+    ALTER TABLE pipelines
+      ADD COLUMN search tsvector
+      GENERATED ALWAYS AS (
+        setweight(to_tsvector('english', coalesce(slug, '')), 'A') ||
+        setweight(to_tsvector('english', coalesce(immutable_array_to_string(tags, ' '), '')), 'B') ||
+        setweight(to_tsvector('english', coalesce(description, '')), 'C')
+      ) STORED
+    """
+
+    execute("CREATE INDEX pipelines_trgm_idx ON pipelines USING GIN (search)")
+  end
+end

test/memex/contexts_test.exs

@@ -17,6 +17,36 @@ defmodule Memex.ContextsTest do
       assert Contexts.list_contexts(user) == [context_a, context_b, context_c]
     end
 
+    test "list_contexts/2 returns relevant contexts for a user", %{user: user} do
+      context_a = context_fixture(%{slug: "dogs", content: "has some treats in it"}, user)
+      context_b = context_fixture(%{slug: "cats", tags: ["home"]}, user)
+
+      context_c =
+        %{slug: "chickens", content: "bananas stuff", tags: ["life", "decisions"]}
+        |> context_fixture(user)
+
+      _shouldnt_return =
+        %{slug: "dog", content: "banana treat stuff", visibility: :private}
+        |> context_fixture(user_fixture())
+
+      # slug
+      assert Contexts.list_contexts("dog", user) == [context_a]
+      assert Contexts.list_contexts("dogs", user) == [context_a]
+      assert Contexts.list_contexts("cat", user) == [context_b]
+      assert Contexts.list_contexts("chicken", user) == [context_c]
+
+      # content
+      assert Contexts.list_contexts("treat", user) == [context_a]
+      assert Contexts.list_contexts("banana", user) == [context_c]
+      assert Contexts.list_contexts("stuff", user) == [context_c]
+
+      # tag
+      assert Contexts.list_contexts("home", user) == [context_b]
+      assert Contexts.list_contexts("life", user) == [context_c]
+      assert Contexts.list_contexts("decision", user) == [context_c]
+      assert Contexts.list_contexts("decisions", user) == [context_c]
+    end
+
     test "list_public_contexts/0 returns public contexts", %{user: user} do
       public_context = context_fixture(%{visibility: :public}, user)
       context_fixture(%{visibility: :unlisted}, user)
@@ -24,6 +54,51 @@ defmodule Memex.ContextsTest do
       assert Contexts.list_public_contexts() == [public_context]
     end
 
+    test "list_public_contexts/1 returns relevant contexts for a user", %{user: user} do
+      context_a =
+        %{slug: "dogs", content: "has some treats in it", visibility: :public}
+        |> context_fixture(user)
+
+      context_b =
+        %{slug: "cats", tags: ["home"], visibility: :public}
+        |> context_fixture(user)
+
+      context_c =
+        %{
+          slug: "chickens",
+          content: "bananas stuff",
+          tags: ["life", "decisions"],
+          visibility: :public
+        }
+        |> context_fixture(user)
+
+      _shouldnt_return =
+        %{
+          slug: "dog",
+          content: "treats bananas stuff",
+          tags: ["home", "life", "decisions"],
+          visibility: :private
+        }
+        |> context_fixture(user)
+
+      # slug
+      assert Contexts.list_public_contexts("dog") == [context_a]
+      assert Contexts.list_public_contexts("dogs") == [context_a]
+      assert Contexts.list_public_contexts("cat") == [context_b]
+      assert Contexts.list_public_contexts("chicken") == [context_c]
+
+      # content
+      assert Contexts.list_public_contexts("treat") == [context_a]
+      assert Contexts.list_public_contexts("banana") == [context_c]
+      assert Contexts.list_public_contexts("stuff") == [context_c]
+
+      # tag
+      assert Contexts.list_public_contexts("home") == [context_b]
+      assert Contexts.list_public_contexts("life") == [context_c]
+      assert Contexts.list_public_contexts("decision") == [context_c]
+      assert Contexts.list_public_contexts("decisions") == [context_c]
+    end
+
     test "get_context!/1 returns the context with given id", %{user: user} do
       context = context_fixture(%{visibility: :public}, user)
       assert Contexts.get_context!(context.id, user) == context

test/memex/notes_test.exs

@@ -14,9 +14,41 @@ defmodule Memex.NotesTest do
       note_a = note_fixture(%{slug: "a", visibility: :public}, user)
       note_b = note_fixture(%{slug: "b", visibility: :unlisted}, user)
       note_c = note_fixture(%{slug: "c", visibility: :private}, user)
+      _shouldnt_return = note_fixture(%{visibility: :private}, user_fixture())
+
       assert Notes.list_notes(user) == [note_a, note_b, note_c]
     end
 
+    test "list_notes/2 returns relevant notes for a user", %{user: user} do
+      note_a = note_fixture(%{slug: "dogs", content: "has some treats in it"}, user)
+      note_b = note_fixture(%{slug: "cats", tags: ["home"]}, user)
+
+      note_c =
+        %{slug: "chickens", content: "bananas stuff", tags: ["life", "decisions"]}
+        |> note_fixture(user)
+
+      _shouldnt_return =
+        %{slug: "dog", content: "banana treat stuff", visibility: :private}
+        |> note_fixture(user_fixture())
+
+      # slug
+      assert Notes.list_notes("dog", user) == [note_a]
+      assert Notes.list_notes("dogs", user) == [note_a]
+      assert Notes.list_notes("cat", user) == [note_b]
+      assert Notes.list_notes("chicken", user) == [note_c]
+
+      # content
+      assert Notes.list_notes("treat", user) == [note_a]
+      assert Notes.list_notes("banana", user) == [note_c]
+      assert Notes.list_notes("stuff", user) == [note_c]
+
+      # tag
+      assert Notes.list_notes("home", user) == [note_b]
+      assert Notes.list_notes("life", user) == [note_c]
+      assert Notes.list_notes("decision", user) == [note_c]
+      assert Notes.list_notes("decisions", user) == [note_c]
+    end
+
     test "list_public_notes/0 returns public notes", %{user: user} do
       public_note = note_fixture(%{visibility: :public}, user)
       note_fixture(%{visibility: :unlisted}, user)
@@ -24,6 +56,51 @@ defmodule Memex.NotesTest do
       assert Notes.list_public_notes() == [public_note]
     end
 
+    test "list_public_notes/1 returns relevant notes for a user", %{user: user} do
+      note_a =
+        %{slug: "dogs", content: "has some treats in it", visibility: :public}
+        |> note_fixture(user)
+
+      note_b =
+        %{slug: "cats", tags: ["home"], visibility: :public}
+        |> note_fixture(user)
+
+      note_c =
+        %{
+          slug: "chickens",
+          content: "bananas stuff",
+          tags: ["life", "decisions"],
+          visibility: :public
+        }
+        |> note_fixture(user)
+
+      _shouldnt_return =
+        %{
+          slug: "dog",
+          content: "treats bananas stuff",
+          tags: ["home", "life", "decisions"],
+          visibility: :private
+        }
+        |> note_fixture(user)
+
+      # slug
+      assert Notes.list_public_notes("dog") == [note_a]
+      assert Notes.list_public_notes("dogs") == [note_a]
+      assert Notes.list_public_notes("cat") == [note_b]
+      assert Notes.list_public_notes("chicken") == [note_c]
+
+      # content
+      assert Notes.list_public_notes("treat") == [note_a]
+      assert Notes.list_public_notes("banana") == [note_c]
+      assert Notes.list_public_notes("stuff") == [note_c]
+
+      # tag
+      assert Notes.list_public_notes("home") == [note_b]
+      assert Notes.list_public_notes("life") == [note_c]
+      assert Notes.list_public_notes("decision") == [note_c]
+      assert Notes.list_public_notes("decisions") == [note_c]
+    end
+
     test "get_note!/1 returns the note with given id", %{user: user} do
       note = note_fixture(%{visibility: :public}, user)
       assert Notes.get_note!(note.id, user) == note

test/memex/pipelines_test.exs

@@ -17,6 +17,36 @@ defmodule Memex.PipelinesTest do
       assert Pipelines.list_pipelines(user) == [pipeline_a, pipeline_b, pipeline_c]
     end
 
+    test "list_pipelines/2 returns relevant pipelines for a user", %{user: user} do
+      pipeline_a = pipeline_fixture(%{slug: "dogs", description: "has some treats in it"}, user)
+      pipeline_b = pipeline_fixture(%{slug: "cats", tags: ["home"]}, user)
+
+      pipeline_c =
+        %{slug: "chickens", description: "bananas stuff", tags: ["life", "decisions"]}
+        |> pipeline_fixture(user)
+
+      _shouldnt_return =
+        %{slug: "dog", description: "banana treat stuff", visibility: :private}
+        |> pipeline_fixture(user_fixture())
+
+      # slug
+      assert Pipelines.list_pipelines("dog", user) == [pipeline_a]
+      assert Pipelines.list_pipelines("dogs", user) == [pipeline_a]
+      assert Pipelines.list_pipelines("cat", user) == [pipeline_b]
+      assert Pipelines.list_pipelines("chicken", user) == [pipeline_c]
+
+      # description
+      assert Pipelines.list_pipelines("treat", user) == [pipeline_a]
+      assert Pipelines.list_pipelines("banana", user) == [pipeline_c]
+      assert Pipelines.list_pipelines("stuff", user) == [pipeline_c]
+
+      # tag
+      assert Pipelines.list_pipelines("home", user) == [pipeline_b]
+      assert Pipelines.list_pipelines("life", user) == [pipeline_c]
+      assert Pipelines.list_pipelines("decision", user) == [pipeline_c]
+      assert Pipelines.list_pipelines("decisions", user) == [pipeline_c]
+    end
+
     test "list_public_pipelines/0 returns public pipelines", %{user: user} do
       public_pipeline = pipeline_fixture(%{visibility: :public}, user)
       pipeline_fixture(%{visibility: :unlisted}, user)
@@ -24,6 +54,51 @@ defmodule Memex.PipelinesTest do
       assert Pipelines.list_public_pipelines() == [public_pipeline]
     end
 
+    test "list_public_pipelines/1 returns relevant pipelines for a user", %{user: user} do
+      pipeline_a =
+        %{slug: "dogs", description: "has some treats in it", visibility: :public}
+        |> pipeline_fixture(user)
+
+      pipeline_b =
+        %{slug: "cats", tags: ["home"], visibility: :public}
+        |> pipeline_fixture(user)
+
+      pipeline_c =
+        %{
+          slug: "chickens",
+          description: "bananas stuff",
+          tags: ["life", "decisions"],
+          visibility: :public
+        }
+        |> pipeline_fixture(user)
+
+      _shouldnt_return =
+        %{
+          slug: "dog",
+          description: "treats bananas stuff",
+          tags: ["home", "life", "decisions"],
+          visibility: :private
+        }
+        |> pipeline_fixture(user)
+
+      # slug
+      assert Pipelines.list_public_pipelines("dog") == [pipeline_a]
+      assert Pipelines.list_public_pipelines("dogs") == [pipeline_a]
+      assert Pipelines.list_public_pipelines("cat") == [pipeline_b]
+      assert Pipelines.list_public_pipelines("chicken") == [pipeline_c]
+
+      # description
+      assert Pipelines.list_public_pipelines("treat") == [pipeline_a]
+      assert Pipelines.list_public_pipelines("banana") == [pipeline_c]
+      assert Pipelines.list_public_pipelines("stuff") == [pipeline_c]
+
+      # tag
+      assert Pipelines.list_public_pipelines("home") == [pipeline_b]
+      assert Pipelines.list_public_pipelines("life") == [pipeline_c]
+      assert Pipelines.list_public_pipelines("decision") == [pipeline_c]
+      assert Pipelines.list_public_pipelines("decisions") == [pipeline_c]
+    end
+
     test "get_pipeline!/1 returns the pipeline with given id", %{user: user} do
       pipeline = pipeline_fixture(%{visibility: :public}, user)
       assert Pipelines.get_pipeline!(pipeline.id, user) == pipeline

test/support/fixtures/contexts_fixtures.ex

@@ -16,7 +16,7 @@ defmodule Memex.ContextsFixtures do
       attrs
       |> Enum.into(%{
         content: "some content",
-        tag: [],
+        tags: [],
         slug: random_slug(),
         visibility: :private
       })

test/support/fixtures/notes_fixtures.ex

@@ -16,7 +16,7 @@ defmodule Memex.NotesFixtures do
       attrs
       |> Enum.into(%{
         content: "some content",
-        tag: [],
+        tags: [],
         slug: random_slug(),
         visibility: :private
       })

test/support/fixtures/pipelines_fixtures.ex

@@ -16,7 +16,7 @@ defmodule Memex.PipelinesFixtures do
       attrs
       |> Enum.into(%{
         description: "some description",
-        tag: [],
+        tags: [],
         slug: random_slug(),
         visibility: :private
       })