From 27d63abfe88f478a0b5a2cf2b3345072fb5bfdde Mon Sep 17 00:00:00 2001 From: dev_agent Date: Sun, 11 Oct 2026 10:12:15 -0400 Subject: [PATCH] Clear every name the old read rule withheld in migration 0020 The first cut cleared only dotted names spelled in [A-Za-z0-9.:-], which missed host names the old rule withheld: ones with "_" or non-ASCII characters, and a dotless name equal to the link's host. Since SourceOf now shows a stored name as it stands, 0020 must clear a superset: any name without a space that is dotted, holds a colon, or appears in the recipe's own link. Maestro 966066e3-0e25-4b8b-9bb0-0694c47c9106 Co-Authored-By: Claude Opus 5.5 --- internal/store/migrate_test.go | 38 ++++++++++++++----------- migrations/0020_site_name_from_page.sql | 29 +++++++++++-------- 2 files changed, 39 insertions(+), 28 deletions(-) diff --git a/internal/store/migrate_test.go b/internal/store/migrate_test.go index 0559561..e026ae0 100644 --- a/internal/store/migrate_test.go +++ b/internal/store/migrate_test.go @@ -926,35 +926,41 @@ func TestMigrateShoppingListChatSendUnconfirmed(t *testing.T) { // TestMigrateSiteNameFromPage applies 0020 over recipes and idea job // results stored when a page naming no site was stored under its host: a -// stored name that could be a host — a dotted host name or an IP address — -// is cleared, so every stored name is one a page gave; any other name, -// including a dotted one with spaces, is kept, as is everything else in -// the rows. +// stored name that could be a host — one with no space that is dotted, an +// IP address or spelled in the recipe's own link — is cleared, so every +// stored name is one a page gave; a name with a space is kept, as is +// everything else in the rows. func TestMigrateSiteNameFromPage(t *testing.T) { ctx := context.Background() s := openTestStore(t) migrateBefore(t, s, "0020") sites := []struct { - stored, want any + stored any + link string + want any }{ - {"private-token.example.com", nil}, - {"Food.com", nil}, - {"203.0.113.7", nil}, - {"2001:db8::1", nil}, - {"Curry House", "Curry House"}, - {"Delish", "Delish"}, - {"Mr. Food Test Kitchen", "Mr. Food Test Kitchen"}, - {nil, nil}, + {"private-token.example.com", "https://private-token.example.com/stew", nil}, + {"redirect_token.example.net", "https://kitchen.example/stew", nil}, + {"bücher.example", "https://kitchen.example/stew", nil}, + {"Food.com", "https://www.food.com/stew", nil}, + {"203.0.113.7", "https://203.0.113.7/stew", nil}, + {"2001:db8::1", "https://[2001:db8::1]/stew", nil}, + {"TokenHost", "https://www.tokenhost/stew", nil}, + {"Delish", "https://www.delish.com/stew", nil}, + {"Delish", "https://kitchen.example/stew", "Delish"}, + {"Curry House", "https://curry-house.example/stew", "Curry House"}, + {"Mr. Food Test Kitchen", "https://kitchen.example/stew", "Mr. Food Test Kitchen"}, + {nil, "https://kitchen.example/stew", nil}, } for i, c := range sites { if _, err := s.Writer.ExecContext(ctx, `INSERT INTO recipes (id, source_url, site_name, title, raw_jsonld, fetched_at, created_at, updated_at) - VALUES (?, ?, ?, 'Stew', '{}', 'f', 'c', 'u')`, fmt.Sprintf("recipe-%d", i), fmt.Sprintf("https://r%d.example/stew", i), c.stored); err != nil { + VALUES (?, ?, ?, 'Stew', '{}', 'f', 'c', 'u')`, fmt.Sprintf("recipe-%d", i), fmt.Sprintf("%s?row=%d", c.link, i), c.stored); err != nil { t.Fatal(err) } } const ( - legacy = `{"stage":"done","ideas":[{"name":"A","site_name":"query-host.example","source_url":"https://query-host.example/a?t=1"},{"name":"B","site_name":"Curry House","source_url":"https://b.example/b"},{"name":"C","site_name":null,"source_url":null}],"dropped":1}` - cleaned = `{"stage":"done","ideas":[{"name":"A","site_name":null,"source_url":"https://query-host.example/a?t=1"},{"name":"B","site_name":"Curry House","source_url":"https://b.example/b"},{"name":"C","site_name":null,"source_url":null}],"dropped":1}` + legacy = `{"stage":"done","ideas":[{"name":"A","site_name":"query-host.example","source_url":"https://query-host.example/a?t=1"},{"name":"B","site_name":"Curry House","source_url":"https://b.example/b"},{"name":"C","site_name":null,"source_url":null},{"name":"D","site_name":"Kitchen","source_url":"https://kitchen.example/d"}],"dropped":1}` + cleaned = `{"stage":"done","ideas":[{"name":"A","site_name":null,"source_url":"https://query-host.example/a?t=1"},{"name":"B","site_name":"Curry House","source_url":"https://b.example/b"},{"name":"C","site_name":null,"source_url":null},{"name":"D","site_name":null,"source_url":"https://kitchen.example/d"}],"dropped":1}` other = `{"ideas":[{"site_name":"host.example"}]}` ) for _, j := range [][3]string{ diff --git a/migrations/0020_site_name_from_page.sql b/migrations/0020_site_name_from_page.sql index 103428c..53b1beb 100644 --- a/migrations/0020_site_name_from_page.sql +++ b/migrations/0020_site_name_from_page.sql @@ -4,28 +4,33 @@ -- only when dinners.SourceOf finds the link safe to show. Before this, a -- page naming no site was stored under the host of the URL it was served -- from, which may carry a personal token, and nothing recorded which a --- stored name was. A stored name that could be a host — a dotted host name --- or an IP address, spelled only in host characters — is cleared here, so --- every stored name is one a page gave. A site that names itself as a --- domain, such as "Food.com", loses its stored name with them until its +-- stored name was. A stored name that could be a host is cleared here, so +-- every stored name is one a page gave: one with no space (a host has +-- none) that is dotted or holds a colon — a host name or an IP address — +-- or that appears in the recipe's own link, as a dotless host does. This +-- clears every name the read rule before it withheld, and more: a site +-- that names itself as a domain, such as "Food.com", or by a word its link +-- spells, such as "Delish" on delish.com, loses its stored name until its -- recipe is fetched again, and is credited by its link's host meanwhile -- when the link is safe. -- -- Idea job results stored before sources were made safe hold the grounded --- recipe's site name as it was stored, and are cleared the same way: an --- idea's stored site name is shown as it stands. The ideas array is --- rebuilt from a plain scan of json_each, which yields its elements in --- array order, so the ideas keep their order (sqlc's parser refuses an --- ORDER BY inside json_group_array). +-- recipe's site name and link as they were stored, and are cleared the +-- same way: an idea's stored site name is shown as it stands. The ideas +-- array is rebuilt from a plain scan of json_each, which yields its +-- elements in array order, so the ideas keep their order (sqlc's parser +-- refuses an ORDER BY inside json_group_array). UPDATE recipes SET site_name = NULL -WHERE site_name GLOB '*[.:]*' AND site_name NOT GLOB '*[^-A-Za-z0-9.:]*'; +WHERE instr(site_name, ' ') = 0 + AND (site_name GLOB '*[.:]*' OR instr(lower(source_url), lower(site_name)) > 0); UPDATE jobs SET result = json_set(result, '$.ideas', ( SELECT json_group_array( CASE - WHEN json_extract(idea.value, '$.site_name') GLOB '*[.:]*' - AND json_extract(idea.value, '$.site_name') NOT GLOB '*[^-A-Za-z0-9.:]*' + WHEN instr(json_extract(idea.value, '$.site_name'), ' ') = 0 + AND (json_extract(idea.value, '$.site_name') GLOB '*[.:]*' + OR instr(lower(json_extract(idea.value, '$.source_url')), lower(json_extract(idea.value, '$.site_name'))) > 0) THEN json_set(idea.value, '$.site_name', json('null')) ELSE json(idea.value) END