Clear every name the old read rule withheld in migration 0020
CI / ci (push) Successful in 8m11s
CI / ci (pull_request) Successful in 8m7s

The first cut cleared only dotted names spelled in [A-Za-z0-9.:-], which
missed host names the old rule withheld: ones with "_" or non-ASCII
characters, and a dotless name equal to the link's host. Since SourceOf
now shows a stored name as it stands, 0020 must clear a superset: any
name without a space that is dotted, holds a colon, or appears in the
recipe's own link.

Maestro 966066e3-0e25-4b8b-9bb0-0694c47c9106

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-11 10:12:15 -04:00
co-authored by Claude Opus 5.5
parent 267d0be4a6
commit 27d63abfe8
2 changed files with 39 additions and 28 deletions
+22 -16
View File
@@ -926,35 +926,41 @@ func TestMigrateShoppingListChatSendUnconfirmed(t *testing.T) {
// TestMigrateSiteNameFromPage applies 0020 over recipes and idea job // TestMigrateSiteNameFromPage applies 0020 over recipes and idea job
// results stored when a page naming no site was stored under its host: a // results stored when a page naming no site was stored under its host: a
// stored name that could be a host — a dotted host name or an IP address — // stored name that could be a host — one with no space that is dotted, an
// is cleared, so every stored name is one a page gave; any other name, // IP address or spelled in the recipe's own link — is cleared, so every
// including a dotted one with spaces, is kept, as is everything else in // stored name is one a page gave; a name with a space is kept, as is
// the rows. // everything else in the rows.
func TestMigrateSiteNameFromPage(t *testing.T) { func TestMigrateSiteNameFromPage(t *testing.T) {
ctx := context.Background() ctx := context.Background()
s := openTestStore(t) s := openTestStore(t)
migrateBefore(t, s, "0020") migrateBefore(t, s, "0020")
sites := []struct { sites := []struct {
stored, want any stored any
link string
want any
}{ }{
{"private-token.example.com", nil}, {"private-token.example.com", "https://private-token.example.com/stew", nil},
{"Food.com", nil}, {"redirect_token.example.net", "https://kitchen.example/stew", nil},
{"203.0.113.7", nil}, {"bücher.example", "https://kitchen.example/stew", nil},
{"2001:db8::1", nil}, {"Food.com", "https://www.food.com/stew", nil},
{"Curry House", "Curry House"}, {"203.0.113.7", "https://203.0.113.7/stew", nil},
{"Delish", "Delish"}, {"2001:db8::1", "https://[2001:db8::1]/stew", nil},
{"Mr. Food Test Kitchen", "Mr. Food Test Kitchen"}, {"TokenHost", "https://www.tokenhost/stew", nil},
{nil, nil}, {"Delish", "https://www.delish.com/stew", nil},
{"Delish", "https://kitchen.example/stew", "Delish"},
{"Curry House", "https://curry-house.example/stew", "Curry House"},
{"Mr. Food Test Kitchen", "https://kitchen.example/stew", "Mr. Food Test Kitchen"},
{nil, "https://kitchen.example/stew", nil},
} }
for i, c := range sites { for i, c := range sites {
if _, err := s.Writer.ExecContext(ctx, `INSERT INTO recipes (id, source_url, site_name, title, raw_jsonld, fetched_at, created_at, updated_at) if _, err := s.Writer.ExecContext(ctx, `INSERT INTO recipes (id, source_url, site_name, title, raw_jsonld, fetched_at, created_at, updated_at)
VALUES (?, ?, ?, 'Stew', '{}', 'f', 'c', 'u')`, fmt.Sprintf("recipe-%d", i), fmt.Sprintf("https://r%d.example/stew", i), c.stored); err != nil { VALUES (?, ?, ?, 'Stew', '{}', 'f', 'c', 'u')`, fmt.Sprintf("recipe-%d", i), fmt.Sprintf("%s?row=%d", c.link, i), c.stored); err != nil {
t.Fatal(err) t.Fatal(err)
} }
} }
const ( const (
legacy = `{"stage":"done","ideas":[{"name":"A","site_name":"query-host.example","source_url":"https://query-host.example/a?t=1"},{"name":"B","site_name":"Curry House","source_url":"https://b.example/b"},{"name":"C","site_name":null,"source_url":null}],"dropped":1}` legacy = `{"stage":"done","ideas":[{"name":"A","site_name":"query-host.example","source_url":"https://query-host.example/a?t=1"},{"name":"B","site_name":"Curry House","source_url":"https://b.example/b"},{"name":"C","site_name":null,"source_url":null},{"name":"D","site_name":"Kitchen","source_url":"https://kitchen.example/d"}],"dropped":1}`
cleaned = `{"stage":"done","ideas":[{"name":"A","site_name":null,"source_url":"https://query-host.example/a?t=1"},{"name":"B","site_name":"Curry House","source_url":"https://b.example/b"},{"name":"C","site_name":null,"source_url":null}],"dropped":1}` cleaned = `{"stage":"done","ideas":[{"name":"A","site_name":null,"source_url":"https://query-host.example/a?t=1"},{"name":"B","site_name":"Curry House","source_url":"https://b.example/b"},{"name":"C","site_name":null,"source_url":null},{"name":"D","site_name":null,"source_url":"https://kitchen.example/d"}],"dropped":1}`
other = `{"ideas":[{"site_name":"host.example"}]}` other = `{"ideas":[{"site_name":"host.example"}]}`
) )
for _, j := range [][3]string{ for _, j := range [][3]string{
+17 -12
View File
@@ -4,28 +4,33 @@
-- only when dinners.SourceOf finds the link safe to show. Before this, a -- only when dinners.SourceOf finds the link safe to show. Before this, a
-- page naming no site was stored under the host of the URL it was served -- page naming no site was stored under the host of the URL it was served
-- from, which may carry a personal token, and nothing recorded which a -- from, which may carry a personal token, and nothing recorded which a
-- stored name was. A stored name that could be a host — a dotted host name -- stored name was. A stored name that could be a host is cleared here, so
-- or an IP address, spelled only in host characters — is cleared here, so -- every stored name is one a page gave: one with no space (a host has
-- every stored name is one a page gave. A site that names itself as a -- none) that is dotted or holds a colon — a host name or an IP address —
-- domain, such as "Food.com", loses its stored name with them until its -- or that appears in the recipe's own link, as a dotless host does. This
-- clears every name the read rule before it withheld, and more: a site
-- that names itself as a domain, such as "Food.com", or by a word its link
-- spells, such as "Delish" on delish.com, loses its stored name until its
-- recipe is fetched again, and is credited by its link's host meanwhile -- recipe is fetched again, and is credited by its link's host meanwhile
-- when the link is safe. -- when the link is safe.
-- --
-- Idea job results stored before sources were made safe hold the grounded -- Idea job results stored before sources were made safe hold the grounded
-- recipe's site name as it was stored, and are cleared the same way: an -- recipe's site name and link as they were stored, and are cleared the
-- idea's stored site name is shown as it stands. The ideas array is -- same way: an idea's stored site name is shown as it stands. The ideas
-- rebuilt from a plain scan of json_each, which yields its elements in -- array is rebuilt from a plain scan of json_each, which yields its
-- array order, so the ideas keep their order (sqlc's parser refuses an -- elements in array order, so the ideas keep their order (sqlc's parser
-- ORDER BY inside json_group_array). -- refuses an ORDER BY inside json_group_array).
UPDATE recipes SET site_name = NULL UPDATE recipes SET site_name = NULL
WHERE site_name GLOB '*[.:]*' AND site_name NOT GLOB '*[^-A-Za-z0-9.:]*'; WHERE instr(site_name, ' ') = 0
AND (site_name GLOB '*[.:]*' OR instr(lower(source_url), lower(site_name)) > 0);
UPDATE jobs SET result = json_set(result, '$.ideas', ( UPDATE jobs SET result = json_set(result, '$.ideas', (
SELECT json_group_array( SELECT json_group_array(
CASE CASE
WHEN json_extract(idea.value, '$.site_name') GLOB '*[.:]*' WHEN instr(json_extract(idea.value, '$.site_name'), ' ') = 0
AND json_extract(idea.value, '$.site_name') NOT GLOB '*[^-A-Za-z0-9.:]*' AND (json_extract(idea.value, '$.site_name') GLOB '*[.:]*'
OR instr(lower(json_extract(idea.value, '$.source_url')), lower(json_extract(idea.value, '$.site_name'))) > 0)
THEN json_set(idea.value, '$.site_name', json('null')) THEN json_set(idea.value, '$.site_name', json('null'))
ELSE json(idea.value) ELSE json(idea.value)
END END