From cb32497043067be285cec17e484c32afaa933c92 Mon Sep 17 00:00:00 2001 From: Holden Thomas Date: Sat, 12 Sep 2026 20:31:54 +0200 Subject: [PATCH 1/8] Implement Wikidata-linked Artsy artwork catalog --- config/config.exs | 5 + config/runtime.exs | 16 + .../artsy-feasibility-2026-09-12.md | 125 +++ docs/integrations/artsy.md | 140 +++ .../absorb/sources/wikidata.ex | 27 +- lib/devils_dictionary/artsy/client.ex | 496 ++++++++++ lib/devils_dictionary/artworks.ex | 395 ++++++++ lib/devils_dictionary/artworks/manifest.ex | 86 ++ lib/devils_dictionary/artworks/seeder.ex | 567 ++++++++++++ .../artworks/wikidata_candidates.ex | 225 +++++ lib/devils_dictionary/discovery/providers.ex | 4 +- .../discovery/providers/artsy.ex | 70 ++ .../components/artwork.ex | 100 ++ .../components/layouts.ex | 3 + .../live/artwork_live.ex | 252 +++++ .../live/connection_live.ex | 20 +- lib/devils_dictionary_web/live/word_live.ex | 35 +- lib/devils_dictionary_web/router.ex | 4 + lib/mix/tasks/dd.artsy.withdraw.ex | 30 + lib/mix/tasks/dd.artworks.seed.ex | 93 ++ priv/artworks/manifests/pilot-v1.json | 864 ++++++++++++++++++ priv/artworks/meaning-mappings-v1.json | 96 ++ .../absorb/sources/wikidata_test.exs | 57 ++ test/devils_dictionary/artsy/client_test.exs | 138 +++ .../artworks/manifest_test.exs | 28 + .../artworks/seeder_test.exs | 207 +++++ .../live/artwork_live_test.exs | 132 +++ .../live/home_live_test.exs | 2 +- 28 files changed, 4209 insertions(+), 8 deletions(-) create mode 100644 docs/integrations/artsy-feasibility-2026-09-12.md create mode 100644 docs/integrations/artsy.md create mode 100644 lib/devils_dictionary/artsy/client.ex create mode 100644 lib/devils_dictionary/artworks.ex create mode 100644 lib/devils_dictionary/artworks/manifest.ex create mode 100644 lib/devils_dictionary/artworks/seeder.ex create mode 100644 lib/devils_dictionary/artworks/wikidata_candidates.ex create mode 100644 lib/devils_dictionary/discovery/providers/artsy.ex create mode 100644 lib/devils_dictionary_web/components/artwork.ex create mode 100644 lib/devils_dictionary_web/live/artwork_live.ex create mode 100644 lib/mix/tasks/dd.artsy.withdraw.ex create mode 100644 lib/mix/tasks/dd.artworks.seed.ex create mode 100644 priv/artworks/manifests/pilot-v1.json create mode 100644 priv/artworks/meaning-mappings-v1.json create mode 100644 test/devils_dictionary/artsy/client_test.exs create mode 100644 test/devils_dictionary/artworks/manifest_test.exs create mode 100644 test/devils_dictionary/artworks/seeder_test.exs create mode 100644 test/devils_dictionary_web/live/artwork_live_test.exs diff --git a/config/config.exs b/config/config.exs index 7bf56cf0..acafba3e 100644 --- a/config/config.exs +++ b/config/config.exs @@ -132,6 +132,11 @@ config :devils_dictionary, :giphy, rating: "g", enabled: true +config :devils_dictionary, :artsy, + endpoint: "https://api.artsy.net", + rate_limit_ms: 340, + enabled: true + # Oban (#69 §5). `absorb: 1` because a dump absorb is a single long stream. # `enrich` is **1**, not the spec's 3: `EnrichWorker` paces with a per-process # `Process.sleep(rate_limit_ms)`, so three concurrent jobs would triple the rate diff --git a/config/runtime.exs b/config/runtime.exs index 069122e2..505f6b66 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -42,6 +42,8 @@ local_provider_env = "CINEGRAPH_API_KEY", "CINEGRAPH_GRAPHQL_URL", "GIPHY_API_KEY", + "ARTSY_CLIENT_ID", + "ARTSY_CLIENT_SECRET", "DISCOVERY_POSITIVE_REFRESH_SECONDS", "DISCOVERY_EMPTY_REFRESH_SECONDS", "CINEGRAPH_DISCOVERY_POSITIVE_REFRESH_SECONDS", @@ -91,6 +93,20 @@ if giphy_api_key = System.get_env("GIPHY_API_KEY") || local_provider_env["GIPHY_ config :devils_dictionary, :giphy, api_key: giphy_api_key end +# Artsy credentials remain server-only. They are read from the ignored local +# development file or exported environment and are never placed in LiveView +# assigns, URLs, logs, manifests, or source records. +artsy_client_id = System.get_env("ARTSY_CLIENT_ID") || local_provider_env["ARTSY_CLIENT_ID"] + +artsy_client_secret = + System.get_env("ARTSY_CLIENT_SECRET") || local_provider_env["ARTSY_CLIENT_SECRET"] + +if artsy_client_id && artsy_client_secret do + config :devils_dictionary, :artsy, + client_id: artsy_client_id, + client_secret: artsy_client_secret +end + parse_policy_integer = fn name -> case System.get_env(name) || local_provider_env[name] do value when value in [nil, ""] -> diff --git a/docs/integrations/artsy-feasibility-2026-09-12.md b/docs/integrations/artsy-feasibility-2026-09-12.md new file mode 100644 index 00000000..8d13a630 --- /dev/null +++ b/docs/integrations/artsy-feasibility-2026-09-12.md @@ -0,0 +1,125 @@ +# Artsy feasibility and open-source comparison — issue #86 + +Research and live-probe date: 2026-09-12. The issue's earlier readiness audit is the +baseline. This report completes the remaining bounded evidence before implementation. +It is a measurement of current API behavior, not a promise of future access or a +license opinion. + +## Exit decision + +**GO for Artsy as optional, disposable discovery and exact-identifier enrichment. +Do not use Artsy as the durable primary collection.** Wikidata supplies cross-source +identity. Independently licensed museum metadata and images, starting with the Met, +are preferred for durable display. Artsy descriptions, gene assignments, image URLs, +and cached responses must remain source-specific and removable. + +The first pilot stays capped at 500 manifest candidates and 500 created/matched +records, with smaller operator-configured request and batch limits. Given the live +search hydration rate, start with Wikidata P11005-linked records rather than broad +title search. Widen only after the pilot reports its actual coverage. + +## Probe bounds and declared sample + +- Earlier issue audit: 39 Artsy requests. +- Completion probe: 93 Artsy requests, including 2 token exchanges; no retries and no + 429 responses. Combined issue evidence: **132 Artsy requests**, below the 200-request + initial-probe ceiling. +- Open-source comparison: 68 Met requests against the current paginated + `/public/collection/v1.1/search` endpoint and object hydration endpoint. +- No image bytes were requested or downloaded. +- Every request used an identifying User-Agent. Secrets and XAPP tokens were held only + in process memory and never printed or written. +- Known-work queries: Mona Lisa; The Starry Night; The Kiss; The Third of May; + Guernica; The Scream; Water Lilies; The Birth of Venus; American Gothic; The Great + Wave. +- Meaning queries: nepotism; war; grief; love; power; family; solitude; justice; bank; + Pop Art. + +## Measured Artsy behavior + +| Capability | Documented | Live result | Implementation consequence | +| --- | --- | --- | --- | +| XAPP authentication | Yes | 2/2 token exchanges returned 201 | Server only; redact credentials/token; refresh once on 401. | +| Artwork-only search | Search is mixed; audit found `type=artwork` behavior | 20/20 declared searches returned 200 and sampled only artwork types | Revalidate response types; the parameter is not trusted as an identity or relevance filter. | +| Search hydration | Search self links may be unavailable | 59 artwork hits; **3 hydrated, 56 returned 404** | A hit is only a candidate. Hydration status is explicit and 404 is useful output, not a crash. | +| Known-work coverage | Not guaranteed | Goya's *The Third of May* hydrated as the expected painting. *The Birth of Venus* was Marcantonio Raimondi's print, not Botticelli's painting. The other canonical title queries produced unavailable or unrelated works. | Never merge or select by title/rank. Preserve reproductions, prints, editions, and originals separately. | +| Usable media references | Artwork resources expose image links and rights strings | 3/3 hydrated search records had thumbnail references and rights strings; image bytes were not tested | Store a source-specific reference and rights statement separately. No file download or permanent-retention assumption. | +| Pagination | Follow `_links.next` | `war` page 1 returned only artworks, but its next link again omitted `type=artwork`; explicitly restoring the filter kept page 2 artwork-only | Parse only allowlisted Artsy URLs, reapply validated immutable filters, and validate every returned type. | +| P11005 exact crosswalk | Wikidata P11005 is an artwork slug | The P11005 example redirected, then returned 200 with slug `rembrandt-van-rijn-the-anatomy-lesson-of-dr-nicolaes-tulp` and opaque ID `4eb063a5b1976400010071f0` | Store `artsy_artwork_slug` and `artsy_artwork_id` in distinct namespaces. Follow only validated redirects. | +| P2042 artist crosswalk | Wikidata P2042 is an artist slug | `vincent-van-gogh` redirected, then returned opaque ID `4d8b92944eb68a1b2c000264` | Resolve creators through exact slug/ID evidence, never through a name string. | +| Direct gene assignments | Artwork links expose assigned genes | Goya returned 5 assignments: 19th Century, Chiaroscuro, Collective History, Conflict, Cultural Commentary | Preserve as attributed Artsy vocabulary. No string-to-meaning equivalence. | +| Gene-filtered acquisition | Gene resources link to artworks | The first 10 `gene_id=19th Century` artwork IDs were identical to the unfiltered first 10 | **Unverified/unsafe.** Do not seed through gene traversal. A direct per-work assignment may label a review candidate only. | +| Retry/quota signal | 5 requests/sec documented | Probe throttled below 3/sec; 0 retries, 0 quota responses | One shared bounded client handles 429/5xx with capped exponential delay and reports exhaustion. | + +The three hydrated search records were: + +1. Goya, *The Third of May* — painting, opaque ID + `4d8b92ee4eb68a1b2c0009ab`, expected known work. +2. Marcantonio Raimondi, *The Birth of Venus* — print, opaque ID + `515bb06394714c2e38001285`, a different work from the famous Botticelli title. +3. Alphonse Legros, *Solitude (Solitude (Paysage))* — print, opaque ID + `515d45217b70570a13003fc1`, meaning-search candidate only. + +No meaning query produced an automatically acceptable semantic connection. The probe +therefore gained **zero reviewed art examples**. `family`, `war`, `grief`, and other +queries may support future human-reviewed candidates; `bank` and title-like queries +demonstrate wrong-sense/creator/reproduction risk. + +## Met comparison over the same 20 queries + +The Met v1.1 highlight-and-image searches returned 48 sampled object records. Of those, +30 exposed a public-domain image reference and 33 had subject tags. Four queries had +no highlight result: Mona Lisa, Guernica, nepotism, and solitude. + +The Met covered a recognizable *Great Wave* object (ID 39799) and supplied strong +open candidates for grief, family, and justice, but it also returned broad or unrelated +matches. For example, `American Gothic` returned a cabinet, *Jubal and Miriam*, and a +pitcher. Open licensing solves durability, not semantic relevance or identity. + +| Dimension | Artsy sample | Met sample | +| --- | ---: | ---: | +| Search/query pages | 20 | 20 | +| Candidate hits hydrated | 3 / 59 | 48 / 48 requested | +| Explicitly usable/open image references | 0 established for permanent use | 30 public-domain image references | +| Subject/classification data | 5 direct genes on the sampled Goya work; traversal unsafe | 33/48 sampled objects had tags | +| Canonical known-work exact-title evidence | 1 expected painting; 1 same-title different print | 1 recognizable Great Wave result; title search remained noisy | +| Permanent retention basis | No; API terms require removable cache/content | Met Open Access/public-domain fields support durable use where the object says so | + +## Retention matrix + +| Field/material | Discovery/cache mode | Durable collection mode | Withdrawal action | +| --- | --- | --- | --- | +| Artsy opaque ID and confirmed slug | Keep as source identifiers while access/terms permit | Keep only if an independent identity source or separate permission supports it | Disable Artsy discovery; remove disallowed projections/aliases as policy requires. | +| Artsy title/date/medium/institution | Short-lived normalized cache | Persist only with independent field-level support or separate permission | Remove Artsy-only field support and recompute display. | +| Artsy description/blurb | Short-lived cache only | No default durable retention | Delete Artsy copies. | +| Artsy genes/assignments | Short-lived attributed cache/candidate evidence | No default durable retention | Delete assignments/mappings and stale any review relying on them. | +| Artsy image URL/rights string | Reference only; no image download | Only when a separately verified license supports the image | Remove Artsy-only reference/copies; fall back to text card or open source. | +| Wikidata QID and P11005/P2042 statements | Versioned source record | Durable under Wikidata licensing/attribution policy | Preserve independently supported identity. | +| Met object metadata | Versioned source record | Durable where the field's Open Access terms apply | Preserve independently supported fields. | +| Met image | Reference or download only when object says public domain | Durable for `isPublicDomain=true`, retaining source/credit metadata | Preserve independently licensed image. | +| Editorial rationale/review | Local first-party content | Durable | Preserve; mark for re-review if cited provider evidence disappears. | + +Artsy's terms permit reasonable freshness-oriented caching, require changes/removal +after notice, and require API content/copies to be removed after termination. API +availability or a public-domain underlying painting does not license Artsy's text, +classification, or image rendition. + +## Meaning and acquisition policy + +- Broad searches such as `family`, `power`, or `patronage` are exploratory and are not + synonyms or evidence for `nepotism`. +- A direct work-gene assignment may generate a clearly labeled candidate for one exact + meaning only through a versioned reviewed mapping. A search hit records the exact + query and is weaker. +- Neither path creates an accepted `illustrates` claim. Existing evidence/review rules + remain authoritative. +- Gene-filter traversal is disabled until a future control query proves it changes the + returned population as documented. + +## Sources + +- [Artsy authentication](https://developers.artsy.net/v2/docs/authentication) +- [Artsy search behavior and unavailable hits](https://developers.artsy.net/v2/docs/search) +- [Artsy artwork links and fields](https://developers.artsy.net/v2/docs/artworks) +- [Artsy API terms](https://developers.artsy.net/v2/terms) +- [Met Collection API, including v1.1 search](https://metmuseum.github.io/) diff --git a/docs/integrations/artsy.md b/docs/integrations/artsy.md new file mode 100644 index 00000000..ffcb4124 --- /dev/null +++ b/docs/integrations/artsy.md @@ -0,0 +1,140 @@ +# Artsy artwork integration + +Status: optional, server-side, bounded, and removable. Implemented for issue +[#86](https://github.com/razrfly/dictionary/issues/86) on top of the shared +source-identity work from issue #93. + +## Product role + +Artsy is a discovery and selected-record enrichment source. It is not the +durable identity authority and it never creates an accepted interpretation. +Wikidata QIDs plus exact P11005 (artwork) and P2042 (artist) statements provide +the durable crosswalk. Artsy's opaque artwork/artist IDs and its human-readable +slugs occupy different namespaces. Labels and years are descriptive fields, +never identity keys. + +The public `/artworks` page searches the local catalog without network calls. +Its separate Artsy form is an explicit, transient source-availability check. +Word pages use only retained direct gene assignments and a versioned set of ten +exact sense/language mappings to display **not-yet-reviewed candidates**. A contributor +selects a local work and exact meaning in the existing `/connect` composer; +normal attribution, evidence, review, revision, and dispute rules apply. + +No Artsy image bytes or hosted derivatives are downloaded. Remote thumbnails +are only reference fields in removable source payloads. Durable encyclopedia +text and images continue to come from independently licensed sources. + +## Configuration + +Set these server environment variables (the values must never be put in a +manifest, browser payload, log, fixture, or committed file): + +```text +ARTSY_CLIENT_ID +ARTSY_CLIENT_SECRET +``` + +If either is absent, the provider is disabled while the local collection keeps +working. The client exchanges them for an in-memory XAPP token, refreshes once +after a 401, follows at most two same-host HTTPS `/api/` redirects, throttles +requests, caps retries, obeys `Retry-After`, and returns scrubbed error codes. + +## Bulk seed runbook + +Build a versioned manifest from exact Wikidata identifiers: + +```sh +mix dd.artworks.seed \ + --manifest priv/artworks/manifests/pilot-v1.json \ + --build --candidate-limit 50 --discovery-request-limit 4 +``` + +Validate without database or provider writes: + +```sh +mix dd.artworks.seed \ + --manifest priv/artworks/manifests/pilot-v1.json \ + --dry-run +``` + +Run or resume a bounded import: + +```sh +mix dd.artworks.seed \ + --manifest priv/artworks/manifests/pilot-v1.json \ + --record-limit 50 --request-limit 160 --batch-size 10 --resume +``` + +The JSON contains a schema version, selection policy, source identifiers, +selection reasons, creator crosswalks, checksum, and per-record outcome. It +contains no credentials. It is saved atomically after each record, so an +interrupted run can resume. Completed records are skipped with `--resume`; +quota-stopped records remain retryable. Re-running without `--resume` is also +safe: exact IDs converge through `SourceIdentity`, source records upsert, and +creator claims use a source-native origin key. + +Only `Painting` Artsy records enter the selected painting catalog. Prints and +other possible reproductions are retained as source observations and reported, +but are not merged into the Wikidata painting. Missing Artsy records still +leave a useful Wikidata/Wikipedia-linked artwork. + +The committed pilot manifest contains 43 paintings across 21 creators found in +two bounded Wikidata requests. It includes such independently identified works +as *The Birth of Venus* (Q151047), *Bacchus and Ariadne* (Q1206860), and +*Portrait of Pope Julius II* (Q11918714). The cap is 5,000 records, while the +issue's intended routine batch is at most 500. + +## Live 2026-09-12 pilot + +The readiness audit made 132 Artsy requests. The subsequent five-record import +made 13: four P11005 slugs were no longer retrievable and one resolved through +a safe redirect, producing one work, one creator link, and 13 direct genes. +The same batch was immediately run again with 13 requests: zero new works, +zero duplicate creator links, one exact match, and the same four honest +unavailable outcomes. There were no retries, conflicts, reproductions, or quota +errors. A final browser search used two more requests and returned an honest +empty state. Readiness, both import runs, and browser validation used 160 Artsy +requests, below the issue's 200-request feasibility ceiling. + +The detailed audit and museum comparison are in +[`artsy-feasibility-2026-09-12.md`](artsy-feasibility-2026-09-12.md). It records +the 10-work/10-meaning probe, unsafe gene-filter finding, pagination defect, +404 rate, exact P11005/P2042 validation, and 20-query Metropolitan Museum +comparison. + +## Retention and shutdown + +Artsy's [API terms](https://developers.artsy.net/v2/terms) do not provide the +independent durable-data license Dictionary needs. Every Artsy source record is +therefore marked removable. Disable new calls by unsetting the credentials or +setting `config :devils_dictionary, :artsy, enabled: false`. + +If access or terms end, execute the explicit withdrawal path: + +```sh +mix dd.artsy.withdraw --reason "API access ended on YYYY-MM-DD" +``` + +It disables the source and mappings, makes cache results undisplayable, retires +materialized outputs, withdraws Artsy-source claims, rejects Artsy-only opaque +IDs, clears provider URLs/hashes, and physically removes Artsy raw revisions. +Wikidata QIDs/P11005 slugs, Wikipedia/Met facts, local identities, and human +editorial claims remain. The operation is transactional and tested. + +## Known limits + +- The Artsy API is retiring and many search hits no longer hydrate. This is + expected source unavailability, not a reason to weaken identity matching. +- Artsy's `gene_id` search filter did not change the sampled result page, so + gene traversal/filter acquisition is disabled. Only direct genes from an + exact hydrated artwork are considered. +- The Metropolitan Museum API remains the better open-image/reference source, + but its broad text search is noisy and four audit concepts had no results. +- The committed manifest is a reproducible seed set, not an assertion that all + 43 provider records remain live. Availability is intentionally recorded per + run. + +Primary API references: [authentication](https://developers.artsy.net/v2/docs/authentication), +[search](https://developers.artsy.net/v2/docs/search), +[artworks](https://developers.artsy.net/v2/docs/artworks), and the +[Metropolitan Museum API](https://metmuseum.github.io/). diff --git a/lib/devils_dictionary/absorb/sources/wikidata.ex b/lib/devils_dictionary/absorb/sources/wikidata.ex index f7a4bc72..bea4fd11 100644 --- a/lib/devils_dictionary/absorb/sources/wikidata.ex +++ b/lib/devils_dictionary/absorb/sources/wikidata.ex @@ -46,7 +46,7 @@ defmodule DevilsDictionary.Absorb.Sources.Wikidata do # The fifteen properties the pipeline reads. Everything else is dropped before # storage; a 130 KB entity becomes a couple of KB. - @keep_properties ~w(P18 P31 P279 P171 P105 P225 P1843 P5063 P8814 P13176 P1420 P910 P373 P846 P685 P345 P4947 P577) + @keep_properties ~w(P18 P31 P279 P171 P105 P225 P1843 P5063 P8814 P13176 P1420 P910 P373 P846 P685 P345 P4947 P577 P170 P2042 P11005) # Parent properties walked to closure. **Only these two**: P171 is the # taxonomy proper and terminates at Animalia, and P13176 is the one hop from @@ -587,6 +587,21 @@ defmodule DevilsDictionary.Absorb.Sources.Wikidata do }) end + def identity_record(%{kind: :person} = row) do + Entry.new(%{ + source_slug: slug(), + object_kind: :entity, + entity_kind: :person, + stable_identifier: %{namespace: "wikidata", external_id: row.qid}, + identifiers: row.external_identifiers, + label: row.label, + description: row.description, + metadata: row.metadata, + eligibility: :eligible, + retention: :durable + }) + end + def identity_record(_row), do: :ignore # English first, then the multilingual label a taxon carries instead, then the @@ -621,6 +636,7 @@ defmodule DevilsDictionary.Absorb.Sources.Wikidata do |> put_if("gbif", Client.string(raw, "P846")) |> put_if("ncbi", Client.string(raw, "P685")) |> put_if("aliases", alias_values(raw)) + |> put_if("wikidata_creators", nonempty(Client.entity_ids(raw, "P170"))) end defp alias_values(raw) do @@ -679,7 +695,9 @@ defmodule DevilsDictionary.Absorb.Sources.Wikidata do } ] ++ external_identifier_values(raw, "P4947", "tmdb_movie", &valid_tmdb_movie_id?/1) ++ - external_identifier_values(raw, "P345", "imdb_title", &valid_imdb_title_id?/1) + external_identifier_values(raw, "P345", "imdb_title", &valid_imdb_title_id?/1) ++ + external_identifier_values(raw, "P11005", "artsy_artwork_slug", &valid_artsy_slug?/1) ++ + external_identifier_values(raw, "P2042", "artsy_artist_slug", &valid_artsy_slug?/1) end defp external_identifier_values(raw, property, namespace, valid?) do @@ -709,6 +727,11 @@ defmodule DevilsDictionary.Absorb.Sources.Wikidata do defp valid_imdb_title_id?(_value), do: false + defp valid_artsy_slug?(value) when is_binary(value), + do: Regex.match?(~r/\A[a-z0-9][a-z0-9-]{1,254}\z/, value) + + defp valid_artsy_slug?(_value), do: false + defp publication_year(raw) do raw |> Client.statements("P577") diff --git a/lib/devils_dictionary/artsy/client.ex b/lib/devils_dictionary/artsy/client.ex new file mode 100644 index 00000000..851b275b --- /dev/null +++ b/lib/devils_dictionary/artsy/client.ex @@ -0,0 +1,496 @@ +defmodule DevilsDictionary.Artsy.Client do + @moduledoc """ + Bounded server-side client for the retiring Artsy public REST API. + + A client value owns one request budget and one in-memory XAPP token. Calls + return the updated client so a Mix import cannot accidentally reset its + counter. Redirects are followed only within `https://api.artsy.net/api/`, + retries are capped, and failures contain stable operator-facing codes without + response bodies, credentials, or tokens. + """ + + @base_url "https://api.artsy.net" + @max_page_size 50 + @default_request_limit 100 + @default_timeout 10_000 + @default_retries 2 + @default_rate_limit_ms 340 + + defstruct base_url: @base_url, + client_id: nil, + client_secret: nil, + token: nil, + token_expires_at: nil, + request_count: 0, + request_limit: @default_request_limit, + retry_count: 0, + max_retries: @default_retries, + timeout_ms: @default_timeout, + rate_limit_ms: @default_rate_limit_ms, + request_fun: nil, + sleep_fun: nil, + now_fun: nil + + @type t :: %__MODULE__{} + + @doc "Creates a client. Missing credentials produce an explicit authentication failure on use." + def new(opts \\ []) do + config = Application.get_env(:devils_dictionary, :artsy, []) + + %__MODULE__{ + base_url: opts[:base_url] || config[:endpoint] || @base_url, + client_id: opts[:client_id] || config[:client_id], + client_secret: opts[:client_secret] || config[:client_secret], + request_limit: positive!(opts[:request_limit] || @default_request_limit, :request_limit), + max_retries: non_negative!(opts[:max_retries] || @default_retries, :max_retries), + timeout_ms: positive!(opts[:timeout_ms] || @default_timeout, :timeout_ms), + rate_limit_ms: + non_negative!( + opts[:rate_limit_ms] || config[:rate_limit_ms] || @default_rate_limit_ms, + :rate_limit_ms + ), + request_fun: opts[:request_fun] || (&Req.request/1), + sleep_fun: opts[:sleep_fun] || (&Process.sleep/1), + now_fun: opts[:now_fun] || (&DateTime.utc_now/0) + } + end + + @doc "True when the server has both credentials. Values are never exposed." + def configured?(%__MODULE__{} = client), + do: present?(client.client_id) and present?(client.client_secret) + + @doc "Hydrates one artwork by opaque API ID or validated slug." + def artwork(client, identifier) do + with :ok <- valid_identifier(identifier) do + authorized(client, :get, "/api/artworks/#{identifier}", []) + else + {:error, reason} -> {:error, failure(reason, nil, "/api/artworks"), client} + end + end + + @doc "Hydrates one artist by opaque API ID or validated slug." + def artist(client, identifier) do + with :ok <- valid_identifier(identifier) do + authorized(client, :get, "/api/artists/#{identifier}", []) + else + {:error, reason} -> {:error, failure(reason, nil, "/api/artists"), client} + end + end + + @doc "Returns every creator on one hydrated artwork page, bounded by `size`." + def artwork_artists(client, artwork_id, opts \\ []) do + size = page_size(opts[:size] || @max_page_size) + authorized(client, :get, "/api/artists", params: [artwork_id: artwork_id, size: size]) + end + + @doc "Returns direct gene assignments for one work. This is not gene traversal." + def artwork_genes(client, artwork_id, opts \\ []) do + size = page_size(opts[:size] || @max_page_size) + authorized(client, :get, "/api/genes", params: [artwork_id: artwork_id, size: size]) + end + + @doc """ + Searches and hydrates artwork hits. + + Artsy's next link is advisory only. The returned offset is parsed after URL + validation, while the caller's `q`, `type=artwork`, and bounded `size` are + rebuilt on the next call. Mixed result types are reported and skipped. + """ + def search_artworks(client, query, opts \\ []) do + query = String.trim(query || "") + size = page_size(opts[:size] || 10) + offset = cursor_offset(opts[:cursor]) + + cond do + query == "" or byte_size(query) > 200 -> + {:error, failure(:invalid_query, nil, "/api/search"), client} + + offset == :error -> + {:error, failure(:invalid_cursor, nil, "/api/search"), client} + + true -> + case authorized(client, :get, "/api/search", + params: [q: query, type: "artwork", size: size, offset: offset] + ) do + {:ok, body, client, meta} -> + hits = get_in(body, ["_embedded", "results"]) || [] + {items, client} = hydrate_hits(client, hits) + next_url = get_in(body, ["_links", "next", "href"]) + + {:ok, + %{ + query: query, + items: items, + mixed_type_count: Enum.count(hits, &(&1["type"] != "artwork")), + next_cursor: next_cursor(next_url, query, size), + returned_next_preserved_filter: query_param(next_url, "type") == "artwork", + request_meta: meta + }, client} + + error -> + error + end + end + end + + @doc "Normalizes the source fields the application is allowed to reason about." + def normalize_artwork(body) when is_map(body) do + %{ + "id" => body["id"], + "slug" => body["slug"], + "title" => body["title"], + "category" => body["category"], + "medium" => body["medium"], + "date" => body["date"], + "collecting_institution" => body["collecting_institution"], + "image_rights" => body["image_rights"], + "thumbnail_url" => link(body, "thumbnail"), + "permalink" => public_link(body, "permalink"), + "artists_url" => api_link(body, "artists"), + "genes_url" => api_link(body, "genes"), + "updated_at" => body["updated_at"], + "unique" => body["unique"] + } + end + + def normalize_artist(body) when is_map(body) do + %{ + "id" => body["id"], + "slug" => body["slug"], + "name" => body["name"], + "birthday" => body["birthday"], + "deathday" => body["deathday"], + "nationality" => body["nationality"], + "permalink" => public_link(body, "permalink") + } + end + + def normalize_gene(body) when is_map(body) do + %{"id" => body["id"], "name" => body["name"], "type" => body["type"]} + end + + @doc "A safe Artsy API URL is HTTPS, on the configured host, under `/api/`." + def safe_api_url?(%__MODULE__{} = client, url) when is_binary(url) do + expected = URI.parse(client.base_url) + uri = URI.parse(url) + + uri.scheme == "https" and uri.host == expected.host and uri.port == expected.port and + String.starts_with?(uri.path || "", "/api/") and is_nil(uri.userinfo) + end + + def safe_api_url?(_client, _url), do: false + + defp hydrate_hits(client, hits) do + Enum.map_reduce(hits, client, fn hit, client -> + cond do + hit["type"] != "artwork" -> + {%{status: :unsupported_type, search: search_summary(hit)}, client} + + not safe_api_url?(client, get_in(hit, ["_links", "self", "href"])) -> + {%{status: :invalid_endpoint, search: search_summary(hit)}, client} + + true -> + href = get_in(hit, ["_links", "self", "href"]) + + case authorized_url(client, :get, href, []) do + {:ok, body, client, meta} -> + {%{ + status: :available, + artwork: normalize_artwork(body), + search: search_summary(hit), + request_meta: meta + }, client} + + {:error, %{status: 404}, client} -> + {%{ + status: :unavailable, + search: search_summary(hit), + failure: "artwork_not_retrievable" + }, client} + + {:error, failure, client} -> + {%{status: :failed, search: search_summary(hit), failure: failure.code}, client} + end + end + end) + end + + defp search_summary(hit) do + %{ + "title" => hit["title"], + "description" => hit["description"], + "permalink" => public_link(hit, "permalink"), + "thumbnail_url" => link(hit, "thumbnail") + } + end + + defp authorized(client, method, path, opts) do + url = client.base_url <> path + authorized_url(client, method, url, opts) + end + + defp authorized_url(client, method, url, opts) do + with {:ok, client} <- ensure_token(client) do + do_authorized(client, method, url, opts, false) + else + {:error, failure, client} -> {:error, failure, client} + end + end + + defp do_authorized(client, method, url, opts, refreshed?) do + headers = [{"x-xapp-token", client.token}, {"user-agent", user_agent()}] + + case request(client, method, url, Keyword.put(opts, :headers, headers), 0, 0) do + {:ok, %Req.Response{status: status, body: body}, client, meta} + when status in 200..299 and is_map(body) -> + {:ok, body, client, meta} + + {:ok, %Req.Response{status: 401}, client, _meta} when not refreshed? -> + case ensure_token(%{client | token: nil, token_expires_at: nil}) do + {:ok, client} -> do_authorized(client, method, url, opts, true) + {:error, failure, client} -> {:error, failure, client} + end + + {:ok, %Req.Response{status: status}, client, meta} -> + {:error, failure(http_code(status), status, URI.parse(url).path, meta), client} + + {:error, failure, client} -> + {:error, failure, client} + end + end + + defp ensure_token(%__MODULE__{} = client) do + cond do + token_fresh?(client) -> + {:ok, client} + + not configured?(client) -> + {:error, failure(:credentials_missing, nil, "/api/tokens/xapp_token"), client} + + true -> + opts = [ + json: %{client_id: client.client_id, client_secret: client.client_secret}, + headers: [{"user-agent", user_agent()}] + ] + + case request(client, :post, client.base_url <> "/api/tokens/xapp_token", opts, 0, 0) do + {:ok, %Req.Response{status: status, body: %{"token" => token} = body}, client, _meta} + when status in [200, 201] and is_binary(token) -> + {:ok, + %{ + client + | token: token, + token_expires_at: parse_datetime(body["expires_at"]) + }} + + {:ok, %Req.Response{status: status}, client, meta} -> + {:error, failure(:authentication_failed, status, "/api/tokens/xapp_token", meta), + client} + + {:error, failure, client} -> + {:error, %{failure | code: "authentication_unavailable"}, client} + end + end + end + + defp request(client, method, url, opts, retry_number, redirects) do + if client.request_count >= client.request_limit do + {:error, failure(:request_limit, nil, URI.parse(url).path), client} + else + client.sleep_fun.(client.rate_limit_ms) + client = %{client | request_count: client.request_count + 1} + + request_opts = + [ + method: method, + url: url, + redirect: false, + retry: false, + receive_timeout: client.timeout_ms, + connect_options: [timeout: client.timeout_ms] + ] ++ opts + + case client.request_fun.(request_opts) do + {:ok, %Req.Response{status: status} = response} + when status in [429, 500, 502, 503, 504] and retry_number < client.max_retries -> + delay = retry_delay(response, retry_number) + client.sleep_fun.(delay) + + request( + %{client | retry_count: client.retry_count + 1}, + method, + url, + opts, + retry_number + 1, + redirects + ) + + {:ok, %Req.Response{status: status} = response} + when status in 300..399 and redirects < 2 -> + case redirect_url(response) do + redirect when is_binary(redirect) -> + if safe_api_url?(client, redirect) do + request( + client, + :get, + redirect, + Keyword.delete(opts, :json), + retry_number, + redirects + 1 + ) + else + {:error, failure(:unsafe_redirect, status, URI.parse(url).path), client} + end + + _ -> + {:ok, response, client, request_meta(retry_number, redirects)} + end + + {:ok, response} -> + {:ok, response, client, request_meta(retry_number, redirects)} + + {:error, _reason} when retry_number < client.max_retries -> + client.sleep_fun.(retry_delay(nil, retry_number)) + + request( + %{client | retry_count: client.retry_count + 1}, + method, + url, + opts, + retry_number + 1, + redirects + ) + + {:error, reason} -> + {:error, + failure( + transport_code(reason), + nil, + URI.parse(url).path, + request_meta(retry_number, redirects) + ), client} + end + end + end + + defp token_fresh?(%{token: token, token_expires_at: nil}), do: present?(token) + + defp token_fresh?(%{token: token, token_expires_at: expires, now_fun: now_fun}) do + present?(token) and DateTime.compare(expires, DateTime.add(now_fun.(), 60, :second)) == :gt + end + + defp next_cursor(url, query, _size) do + with true <- is_binary(url), + uri <- URI.parse(url), + true <- + uri.scheme == "https" and uri.host == "api.artsy.net" and uri.path == "/api/search", + params <- URI.decode_query(uri.query || ""), + true <- params["q"] == query, + {offset, ""} when offset >= 0 <- Integer.parse(params["offset"] || "") do + Integer.to_string(offset) + else + _ -> nil + end + end + + defp cursor_offset(nil), do: 0 + defp cursor_offset(value) when is_integer(value) and value >= 0, do: value + + defp cursor_offset(value) when is_binary(value) do + case Integer.parse(value) do + {integer, ""} when integer >= 0 and integer <= 10_000 -> integer + _ -> :error + end + end + + defp cursor_offset(_), do: :error + defp page_size(size) when is_integer(size), do: size |> max(1) |> min(@max_page_size) + defp page_size(_), do: 10 + + defp valid_identifier(value) when is_binary(value) do + if Regex.match?(~r/\A[a-z0-9][a-z0-9-]{1,254}\z/, value), + do: :ok, + else: {:error, :invalid_identifier} + end + + defp valid_identifier(_), do: {:error, :invalid_identifier} + + defp public_link(body, name) do + case link(body, name) do + url when is_binary(url) -> + uri = URI.parse(url) + if uri.scheme == "https" and uri.host in ["www.artsy.net", "artsy.net"], do: url + + _ -> + nil + end + end + + defp api_link(body, name) do + case link(body, name) do + "https://api.artsy.net/api/" <> _ = url -> url + _ -> nil + end + end + + defp link(body, name), do: get_in(body, ["_links", name, "href"]) + + defp redirect_url(response) do + case Req.Response.get_header(response, "location") do + [url | _] -> url + _ -> get_in(response.body, ["_links", "location", "href"]) + end + end + + defp query_param(nil, _key), do: nil + + defp query_param(url, key), + do: URI.parse(url).query |> then(&URI.decode_query(&1 || "")) |> Map.get(key) + + defp retry_delay(response, retry_number) do + retry_after = + if response, do: DevilsDictionary.Discovery.Transport.retry_after_seconds(response) + + min((retry_after || trunc(:math.pow(2, retry_number))) * 1_000, 10_000) + end + + defp request_meta(retries, redirects), do: %{retries: retries, redirects: redirects} + + defp parse_datetime(value) when is_binary(value) do + case DateTime.from_iso8601(value) do + {:ok, datetime, _offset} -> datetime + _ -> nil + end + end + + defp parse_datetime(_), do: nil + + defp http_code(401), do: :authentication_failed + defp http_code(403), do: :access_forbidden + defp http_code(404), do: :not_found + defp http_code(429), do: :quota_exhausted + defp http_code(status) when status >= 500, do: :provider_unavailable + defp http_code(_), do: :http_error + + defp transport_code(%Req.TransportError{reason: :timeout}), do: :timeout + defp transport_code(_), do: :provider_unavailable + + defp failure(code, status, path, meta \\ %{}) do + %{ + code: to_string(code), + status: status, + path: path, + retries: meta[:retries] || 0, + redirects: meta[:redirects] || 0 + } + end + + defp user_agent, do: Application.fetch_env!(:devils_dictionary, :user_agent) + defp present?(value) when is_binary(value), do: String.trim(value) != "" + defp present?(_), do: false + + defp positive!(value, _name) when is_integer(value) and value > 0, do: value + defp positive!(_value, name), do: raise(ArgumentError, "#{name} must be positive") + defp non_negative!(value, _name) when is_integer(value) and value >= 0, do: value + defp non_negative!(_value, name), do: raise(ArgumentError, "#{name} must be non-negative") +end diff --git a/lib/devils_dictionary/artworks.ex b/lib/devils_dictionary/artworks.ex new file mode 100644 index 00000000..517cbeaa --- /dev/null +++ b/lib/devils_dictionary/artworks.ex @@ -0,0 +1,395 @@ +defmodule DevilsDictionary.Artworks do + @moduledoc "Local reusable artwork catalog, source-specific details, meaning candidates, and withdrawal." + + import Ecto.Query + + alias DevilsDictionary.Claims + alias DevilsDictionary.Claims.{Assertion, AssertionEvidence, AssertionRevision} + alias DevilsDictionary.Corpus.SourceRecordRevision + alias DevilsDictionary.Discovery.{Mapping, Result, Run} + alias DevilsDictionary.Encyclopedia + + alias DevilsDictionary.Registry.{ + Entity, + ExternalIdentifier, + Object, + Sense, + SenseRevision, + WorkDetails + } + + alias DevilsDictionary.Repo + alias DevilsDictionary.Sources.{Actor, MaterializedOutput, Source, SourceRecord} + + @catalog_limit 50 + @candidate_scan_limit 500 + @mapping_path Application.app_dir(:devils_dictionary, "priv/artworks/meaning-mappings-v1.json") + + @doc "Searches reusable local artwork identities by title or creator name. No provider call." + def search(query \\ "", opts \\ []) do + query = String.trim(query || "") + limit = opts[:limit] || @catalog_limit + pattern = "%#{escape_like(query)}%" + + base = + from entity in Entity, + join: object in Object, + on: object.id == entity.object_id and object.lifecycle_state == :active, + join: details in WorkDetails, + on: details.entity_id == entity.object_id and details.work_kind == "artwork", + left_join: revision in AssertionRevision, + on: revision.subject_object_id == entity.object_id and revision.is_current, + left_join: predicate in Claims.Predicate, + on: predicate.id == revision.predicate_id and predicate.key == "authored_by", + left_join: creator in Entity, + on: creator.object_id == revision.object_object_id, + where: + ^(query == "") or ilike(entity.preferred_label, ^pattern) or + ilike(creator.preferred_label, ^pattern), + distinct: entity.object_id, + order_by: [asc: entity.preferred_label, asc: entity.object_id], + limit: ^limit, + select: entity + + entities = Repo.all(base) + views(entities) + end + + def get(object_id) when is_integer(object_id) do + case Repo.one( + from entity in Entity, + join: object in Object, + on: object.id == entity.object_id and object.lifecycle_state == :active, + join: details in WorkDetails, + on: details.entity_id == entity.object_id and details.work_kind == "artwork", + where: entity.object_id == ^object_id, + select: entity + ) do + nil -> nil + entity -> List.first(views([entity])) + end + end + + @doc "Returns local, exact-sense suggestions from direct retained Artsy gene assignments." + def suggestions(lexeme_ids) when is_list(lexeme_ids) do + mappings = meaning_mappings() + + senses = + Repo.all( + from sense in Sense, + join: revision in SenseRevision, + on: revision.sense_id == sense.object_id and revision.is_current, + join: lexeme in DevilsDictionary.Registry.Lexeme, + on: lexeme.object_id == sense.lexeme_id, + where: sense.lexeme_id in ^lexeme_ids and sense.identity_state == :active, + select: %{ + id: sense.object_id, + lemma: lexeme.lemma, + language: lexeme.language_tag, + gloss: revision.gloss + } + ) + + assignments = active_artsy_assignments() + catalog = Map.new(search("", limit: @candidate_scan_limit), &{&1.object_id, &1}) + + for sense <- senses, + mapping <- mappings, + mapping_matches?(mapping, sense), + assignment <- assignments, + gene_matches?(mapping, assignment.genes), + artwork = catalog[assignment.object_id], + not is_nil(artwork) do + %{ + artwork: artwork, + sense_id: sense.id, + meaning: sense.gloss, + language: sense.language, + match_type: mapping["match_type"], + match_reason: %{ + provider: "Artsy", + kind: "direct_gene_assignment", + gene_id: mapping["gene_id"], + gene_name: mapping["gene_name"], + mapping_version: mapping["version"], + note: mapping["note"] + }, + review_state: :not_yet_reviewed + } + end + |> Enum.uniq_by(&{&1.artwork.object_id, &1.sense_id, &1.match_reason.gene_id}) + end + + @doc """ + Disables Artsy and removes its disallowed payloads and projections. + + Independently supported objects, Wikidata P11005 slugs, Met/Wikidata fields, + and editorial claims remain. Artsy-only opaque IDs are rejected, Artsy source + claims are withdrawn, cache rows become undisplayable, and raw revisions are + physically removed. + """ + def withdraw_artsy(reason) when is_binary(reason) and byte_size(reason) > 0 do + Repo.transaction(fn -> + source = Repo.one!(from source in Source, where: source.slug == "artsy", lock: "FOR UPDATE") + actor = withdrawal_actor!() + now = DateTime.utc_now() + + record_ids = + Repo.all( + from record in SourceRecord, where: record.source_id == ^source.id, select: record.id + ) + + revision_ids = + Repo.all( + from revision in SourceRecordRevision, + where: revision.source_record_id in ^record_ids, + select: revision.id + ) + + identifiers = + Repo.all( + from identifier in ExternalIdentifier, + where: + identifier.source_record_revision_id in ^revision_ids and + identifier.namespace in ["artsy_artwork_id", "artsy_artist_id"], + lock: "FOR UPDATE" + ) + + Enum.each(identifiers, fn identifier -> + identifier + |> ExternalIdentifier.changeset(%{ + status: :rejected, + source_record_revision_id: nil, + metadata: + Map.merge(identifier.metadata || %{}, %{ + "withdrawn_source" => "artsy", + "withdrawal_reason" => reason, + "withdrawn_at" => DateTime.to_iso8601(now) + }) + }) + |> Repo.update!() + end) + + claim_ids = + Repo.all( + from assertion in Assertion, + join: revision in AssertionRevision, + on: revision.assertion_id == assertion.id and revision.is_current, + where: assertion.source_id == ^source.id and revision.lifecycle_state == :active, + select: assertion.id + ) + + Enum.each(claim_ids, &Claims.withdraw(&1, reason: reason)) + + # Editorial claims are independently retainable, but an Artsy citation + # is not. Remove those evidence edges before deleting the raw revisions; + # source-owned assertions were already withdrawn above. + {evidence_removed, _} = + Repo.delete_all( + from evidence in AssertionEvidence, + where: evidence.source_record_revision_id in ^revision_ids + ) + + {outputs, _} = + Repo.update_all( + from(output in MaterializedOutput, where: output.source_record_id in ^record_ids), + set: [retired_at: now, updated_at: now] + ) + + {records, _} = + Repo.update_all( + from(record in SourceRecord, where: record.id in ^record_ids), + set: [ + display_allowed: false, + display_policy_reason: reason, + display_policy_changed_at: now, + display_policy_actor_id: actor.id, + url: nil, + content_hash: nil, + updated_at: now + ] + ) + + Repo.update_all(from(mapping in Mapping, where: mapping.source_id == ^source.id), + set: [enabled: false, updated_at: now] + ) + + Repo.update_all( + from(run in Run, + join: mapping in Mapping, + on: mapping.id == run.mapping_id, + where: mapping.source_id == ^source.id + ), + set: [display_allowed: false, updated_at: now] + ) + + Repo.update_all( + from(result in Result, + join: run in Run, + on: run.id == result.run_id, + join: mapping in Mapping, + on: mapping.id == run.mapping_id, + where: mapping.source_id == ^source.id + ), + set: [display_allowed: false, updated_at: now] + ) + + {payloads, _} = + Repo.delete_all( + from revision in SourceRecordRevision, where: revision.id in ^revision_ids + ) + + source + |> Source.changeset(%{ + active: false, + discovery_retry_after: nil, + discovery_retry_reason: reason, + config: + Map.merge(source.config || %{}, %{ + "withdrawn_at" => DateTime.to_iso8601(now), + "withdrawal_reason" => reason + }) + }) + |> Repo.update!() + + %{ + source: "artsy", + records_disabled: records, + payloads_deleted: payloads, + outputs_retired: outputs, + identifiers_rejected: length(identifiers), + claims_withdrawn: length(claim_ids), + evidence_removed: evidence_removed + } + end) + end + + defp views(entities) do + ids = Enum.map(entities, & &1.object_id) + creators = creators(ids) + details = artsy_details(ids) + + Enum.map(entities, fn entity -> + view = Encyclopedia.view(entity) + source = details[entity.object_id] + + %{ + object_id: entity.object_id, + title: entity.preferred_label, + description: entity.description, + image_url: view.image_url, + qid: view.qid, + wikipedia_title: view.wikipedia_title, + creators: Map.get(creators, entity.object_id, []), + artsy: source && source.artwork, + genes: (source && source.genes) || [], + source_record_id: source && source.source_record_id + } + end) + end + + defp creators([]), do: %{} + + defp creators(work_ids) do + Repo.all( + from revision in AssertionRevision, + join: predicate in Claims.Predicate, + on: predicate.id == revision.predicate_id and predicate.key == "authored_by", + join: creator in Entity, + on: creator.object_id == revision.object_object_id, + join: object in Object, + on: object.id == creator.object_id and object.lifecycle_state == :active, + where: + revision.subject_object_id in ^work_ids and revision.is_current and + revision.lifecycle_state == :active, + order_by: [asc: creator.preferred_label, asc: creator.object_id], + select: + {revision.subject_object_id, + %{object_id: creator.object_id, label: creator.preferred_label}} + ) + |> Enum.group_by(&elem(&1, 0), &elem(&1, 1)) + end + + defp artsy_details([]), do: %{} + + defp artsy_details(object_ids) do + Repo.all( + from output in MaterializedOutput, + join: record in SourceRecord, + on: record.id == output.source_record_id, + join: source in Source, + on: source.id == record.source_id and source.slug == "artsy" and source.active, + join: revision in SourceRecordRevision, + on: + revision.source_record_id == record.id and revision.revision_key == record.content_hash, + where: + output.output_object_id in ^object_ids and is_nil(output.retired_at) and + record.display_allowed, + order_by: [desc: revision.observed_at, desc: revision.id], + select: %{ + object_id: output.output_object_id, + source_record_id: record.id, + artwork: revision.payload["artwork"], + genes: revision.payload["genes"] + } + ) + |> Enum.uniq_by(& &1.object_id) + |> Map.new(&{&1.object_id, &1}) + end + + defp active_artsy_assignments do + artsy_details( + Repo.all( + from details in WorkDetails, + join: object in Object, + on: object.id == details.entity_id and object.lifecycle_state == :active, + where: details.work_kind == "artwork", + order_by: details.entity_id, + limit: @candidate_scan_limit, + select: details.entity_id + ) + ) + |> Enum.map(fn {object_id, detail} -> %{object_id: object_id, genes: detail.genes || []} end) + end + + defp meaning_mappings do + @mapping_path |> File.read!() |> Jason.decode!() |> Map.fetch!("mappings") + end + + defp mapping_matches?(mapping, sense) do + mapping["lemma"] == sense.lemma and mapping["language"] == sense.language and + gloss_matches?(mapping["gloss_contains"] || [], sense.gloss) + end + + defp gloss_matches?([], _gloss), do: true + + defp gloss_matches?(terms, gloss) do + down = String.downcase(gloss || "") + Enum.any?(terms, &String.contains?(down, String.downcase(&1))) + end + + defp gene_matches?(mapping, genes) do + Enum.any?(genes, fn gene -> + (is_binary(mapping["gene_id"]) and gene["id"] == mapping["gene_id"]) or + (is_binary(mapping["gene_name"]) and gene["name"] == mapping["gene_name"]) + end) + end + + defp withdrawal_actor! do + attrs = %{ + actor_kind: :import, + label: "Artsy provider withdrawal operation", + metadata: %{"operation" => "provider_withdrawal"} + } + + Repo.get_by(Actor, actor_kind: :import, label: attrs.label) || + Repo.insert!(%Actor{} |> Actor.changeset(attrs)) + end + + defp escape_like(value) do + value + |> String.replace("\\", "\\\\") + |> String.replace("%", "\\%") + |> String.replace("_", "\\_") + end +end diff --git a/lib/devils_dictionary/artworks/manifest.ex b/lib/devils_dictionary/artworks/manifest.ex new file mode 100644 index 00000000..749c5b02 --- /dev/null +++ b/lib/devils_dictionary/artworks/manifest.ex @@ -0,0 +1,86 @@ +defmodule DevilsDictionary.Artworks.Manifest do + @moduledoc "Versioned, credential-free artwork seed manifests with resumable per-record outcomes." + + @schema_version 1 + + def new(candidates, metadata \\ %{}) when is_list(candidates) do + %{ + "schema_version" => @schema_version, + "generated_at" => timestamp(), + "selection" => Map.merge(default_selection(), stringify(metadata)), + "candidates" => + candidates + |> Enum.uniq_by(&{&1["qid"], &1["artsy_artwork_slug"]}) + |> Enum.map(&Map.put_new(&1, "import", %{"status" => "pending"})) + } + |> put_checksum() + end + + def load!(path) do + manifest = path |> File.read!() |> Jason.decode!() + + unless manifest["schema_version"] == @schema_version and is_list(manifest["candidates"]) do + raise ArgumentError, "unsupported artwork seed manifest" + end + + expected = checksum(Map.delete(manifest, "checksum")) + + unless manifest["checksum"] == expected, + do: raise(ArgumentError, "artwork manifest checksum mismatch") + + manifest + end + + def save!(manifest, path) do + manifest = manifest |> Map.put("updated_at", timestamp()) |> put_checksum() + directory = Path.dirname(path) + File.mkdir_p!(directory) + temporary = path <> ".tmp-#{System.unique_integer([:positive])}" + File.write!(temporary, Jason.encode!(manifest, pretty: true) <> "\n") + File.rename!(temporary, path) + manifest + end + + def update_candidate(manifest, index, outcome) when is_integer(index) and is_map(outcome) do + candidates = + List.update_at(manifest["candidates"], index, fn candidate -> + Map.put(candidate, "import", stringify(outcome)) + end) + + manifest |> Map.put("candidates", candidates) |> put_checksum() + end + + def completed?(candidate), + do: + get_in(candidate, ["import", "status"]) in ~w(matched created skipped conflict unavailable) + + def checksum(value) do + :sha256 + |> :crypto.hash(Jason.encode!(value)) + |> Base.encode16(case: :lower) + end + + defp put_checksum(manifest), + do: Map.put(manifest, "checksum", checksum(Map.delete(manifest, "checksum"))) + + defp default_selection do + %{ + "artwork_kinds" => ["painting"], + "identity" => "Wikidata QID plus P11005; title is never identity", + "pilot_cap" => 500, + "wikipedia_required" => false + } + end + + defp stringify(map) do + Map.new(map, fn {key, value} -> {to_string(key), stringify_value(value)} end) + end + + defp stringify_value(value) when is_map(value), do: stringify(value) + defp stringify_value(value) when is_list(value), do: Enum.map(value, &stringify_value/1) + defp stringify_value(value) when is_boolean(value) or is_nil(value), do: value + defp stringify_value(value) when is_atom(value), do: to_string(value) + defp stringify_value(value), do: value + + defp timestamp, do: DateTime.utc_now() |> DateTime.truncate(:second) |> DateTime.to_iso8601() +end diff --git a/lib/devils_dictionary/artworks/seeder.ex b/lib/devils_dictionary/artworks/seeder.ex new file mode 100644 index 00000000..110d9f98 --- /dev/null +++ b/lib/devils_dictionary/artworks/seeder.ex @@ -0,0 +1,567 @@ +defmodule DevilsDictionary.Artworks.Seeder do + @moduledoc "Idempotent, resumable Wikidata-to-Artsy selected artwork importer." + + alias DevilsDictionary.Absorb.Materializer + alias DevilsDictionary.Absorb.Sources.Wikidata + alias DevilsDictionary.Artsy.Client + alias DevilsDictionary.Artworks.Manifest + alias DevilsDictionary.Claims + alias DevilsDictionary.Claims.Assertion + alias DevilsDictionary.Registry + alias DevilsDictionary.SourceIdentity + alias DevilsDictionary.SourceIdentity.Entry + alias DevilsDictionary.Sources + + @default_record_limit 500 + @max_record_limit 5_000 + @default_batch_size 25 + + def run(manifest, opts \\ []) when is_map(manifest) do + dry_run? = Keyword.get(opts, :dry_run, false) + resume? = Keyword.get(opts, :resume, false) + record_limit = bounded(opts[:record_limit] || @default_record_limit, 1, @max_record_limit) + batch_size = bounded(opts[:batch_size] || @default_batch_size, 1, 100) + manifest_path = opts[:manifest_path] + + if dry_run? do + {manifest, + %{ + dry_run: true, + candidates: length(manifest["candidates"]), + eligible: Enum.count(manifest["candidates"], &eligible_candidate?/1), + requests: 0, + next_index: 0 + }} + else + %{sources: sources} = DevilsDictionary.Sources.Catalog.seed!() + artsy_source = Map.fetch!(sources, "artsy") + client = opts[:artsy_client] || Client.new(request_limit: opts[:request_limit] || 500) + run = Sources.start_run("dd.artworks.seed", source_id: artsy_source.id) + + initial = %{ + candidates: length(manifest["candidates"]), + processed: 0, + matched: 0, + created: 0, + conflicts: 0, + unavailable: 0, + missing_links: 0, + skipped: 0, + reproductions: 0, + creators_resolved: 0, + creator_links: 0, + genes: 0, + quota_exhausted: false, + requests: 0, + retries: 0, + next_index: nil + } + + try do + {manifest, client, summary} = + manifest["candidates"] + |> Enum.with_index() + |> Enum.chunk_every(batch_size) + |> Enum.reduce_while({manifest, client, initial}, fn batch, acc -> + process_batch(batch, acc, sources, run, resume?, record_limit, manifest_path) + end) + + summary = %{ + summary + | requests: client.request_count, + retries: client.retry_count + } + + Sources.finish_run(run, stringify(summary)) + {maybe_save(manifest, manifest_path), summary} + rescue + exception -> + Sources.fail_run(run, Exception.message(exception)) + reraise exception, __STACKTRACE__ + end + end + end + + defp process_batch( + batch, + {manifest, client, summary}, + sources, + run, + resume?, + record_limit, + path + ) do + result = + Enum.reduce_while(batch, {manifest, client, summary}, fn {candidate, index}, + {manifest, client, summary} -> + cond do + summary.processed >= record_limit -> + {:halt, {manifest, client, %{summary | next_index: index}}} + + resume? and Manifest.completed?(candidate) -> + {:cont, {manifest, client, summary}} + + true -> + {outcome, client} = import_candidate(candidate, client, sources, run) + + manifest = + Manifest.update_candidate( + manifest, + index, + Map.put(outcome, :updated_at, timestamp()) + ) + + summary = accumulate(summary, outcome, index) + manifest = maybe_save(manifest, path) + + if outcome.status == :quota_exhausted do + {:halt, {manifest, client, %{summary | quota_exhausted: true, next_index: index}}} + else + {:cont, {manifest, client, summary}} + end + end + end) + + case result do + {_manifest, _client, %{quota_exhausted: true}} -> + {:halt, result} + + {_manifest, _client, %{processed: processed}} when processed >= record_limit -> + {:halt, result} + + _ -> + {:cont, result} + end + end + + defp import_candidate(candidate, client, sources, run) do + cond do + not eligible_candidate?(candidate) -> + {%{status: :skipped, reason: "invalid_or_unsupported_candidate"}, client} + + true -> + existing_id = Registry.by_external_id("wikidata", candidate["qid"]) + + {work_id, _wikidata_record} = + ensure_wikidata_work!(candidate, sources["wikidata"], run.id) + + case Client.artwork(client, candidate["artsy_artwork_slug"]) do + {:ok, body, client, request_meta} -> + artwork = Client.normalize_artwork(body) + + import_hydrated( + candidate, + artwork, + work_id, + existing_id, + client, + sources, + run, + request_meta + ) + + {:error, %{code: "not_found"}, client} -> + {%{status: :unavailable, object_id: work_id, reason: "artsy_artwork_not_found"}, + client} + + {:error, %{code: "request_limit"}, client} -> + {%{status: :quota_exhausted, object_id: work_id, reason: "request_limit"}, client} + + {:error, %{code: "quota_exhausted"}, client} -> + {%{status: :quota_exhausted, object_id: work_id, reason: "provider_quota"}, client} + + {:error, failure, client} -> + {%{status: :skipped, object_id: work_id, reason: failure.code}, client} + end + end + end + + defp import_hydrated( + candidate, + artwork, + work_id, + existing_id, + client, + sources, + run, + request_meta + ) do + cond do + artwork["slug"] != candidate["artsy_artwork_slug"] -> + {record, client} = + store_artsy_record(candidate, artwork, [], [], client, sources["artsy"], run.id) + + {:ok, entry} = + Entry.new(%{ + source_slug: "artsy", + source_id: sources["artsy"].id, + source_record_id: record.id, + source_record_revision_id: record.current_revision.id, + import_run_id: run.id, + object_kind: :entity, + entity_kind: :work, + work_kind: "artwork", + stable_identifier: %{namespace: "wikidata", external_id: candidate["qid"]}, + identifiers: [ + %{namespace: "wikidata", external_id: candidate["qid"]}, + %{namespace: "artsy_artwork_slug", external_id: candidate["artsy_artwork_slug"]}, + %{namespace: "artsy_artwork_slug", external_id: artwork["slug"]}, + %{namespace: "artsy_artwork_id", external_id: artwork["id"]} + ], + label: candidate["title"], + eligibility: :eligible, + retention: :durable + }) + + resolution = SourceIdentity.resolve(entry) + + {%{ + status: :conflict, + object_id: work_id, + conflict_id: resolution.conflict_id, + reason: "provider_slug_differs_from_wikidata_p11005", + requested_slug: candidate["artsy_artwork_slug"], + returned_slug: artwork["slug"] + }, client} + + artwork["category"] != "Painting" -> + {record, client} = + store_artsy_record(candidate, artwork, [], [], client, sources["artsy"], run.id) + + {%{ + status: :skipped, + object_id: work_id, + source_record_id: record.id, + reason: "artsy_category_not_painting", + category: artwork["category"], + reproduction: true + }, client} + + true -> + {artists_body, client, artist_failure} = collection(client, :artists, artwork["id"]) + {genes_body, client, gene_failure} = collection(client, :genes, artwork["id"]) + + artists = Enum.map(artists_body, &Client.normalize_artist/1) + genes = Enum.map(genes_body, &Client.normalize_gene/1) + + {record, client} = + store_artsy_record(candidate, artwork, artists, genes, client, sources["artsy"], run.id) + + {:ok, entry} = artsy_entry(candidate, artwork, record, sources["artsy"], run.id) + resolution = SourceIdentity.resolve(entry) + + if resolution.state == :conflicting_identifiers do + {%{ + status: :conflict, + object_id: work_id, + conflict_id: resolution.conflict_id, + reason: resolution.reason + }, client} + else + {creator_stats, client} = + import_creators( + candidate, + artists, + resolution.object_id, + record, + client, + sources, + run + ) + + status = if is_nil(existing_id), do: :created, else: :matched + + {%{ + status: status, + object_id: resolution.object_id, + source_record_id: record.id, + artsy_artwork_id: artwork["id"], + artsy_artwork_slug: artwork["slug"], + creator_count: creator_stats.resolved, + creator_links: creator_stats.links, + missing_creators: creator_stats.missing, + gene_count: length(genes), + artist_failure: artist_failure, + gene_failure: gene_failure, + redirects: request_meta.redirects + }, client} + end + end + end + + defp collection(client, kind, artwork_id) do + result = + case kind do + :artists -> Client.artwork_artists(client, artwork_id) + :genes -> Client.artwork_genes(client, artwork_id) + end + + key = if kind == :artists, do: "artists", else: "genes" + + case result do + {:ok, body, client, _meta} -> {get_in(body, ["_embedded", key]) || [], client, nil} + {:error, failure, client} -> {[], client, failure.code} + end + end + + defp store_artsy_record(candidate, artwork, artists, genes, client, source, run_id) do + raw = %{ + "artwork" => artwork, + "artists" => artists, + "genes" => genes, + "wikidata_qid" => candidate["qid"], + "wikidata_p11005" => candidate["artsy_artwork_slug"], + "selection_reason" => candidate["selection_reason"], + "retention" => %{ + "mode" => "discovery_cache", + "image" => "reference_only", + "durable_source" => "wikidata", + "remove_on_provider_termination" => true + } + } + + {:ok, record} = + Sources.upsert_record(source, %{ + external_id: "artwork:#{artwork["id"]}", + url: artwork["permalink"], + raw: raw, + import_run_id: run_id + }) + + {record, client} + end + + defp artsy_entry(candidate, artwork, record, source, run_id) do + Entry.new(%{ + source_slug: source.slug, + source_id: source.id, + source_record_id: record.id, + source_record_revision_id: record.current_revision.id, + import_run_id: run_id, + object_kind: :entity, + entity_kind: :work, + work_kind: "artwork", + stable_identifier: %{namespace: "artsy_artwork_id", external_id: artwork["id"]}, + identifiers: [ + %{namespace: "wikidata", external_id: candidate["qid"]}, + %{namespace: "artsy_artwork_slug", external_id: artwork["slug"]}, + %{namespace: "artsy_artwork_id", external_id: artwork["id"]} + ], + label: candidate["title"], + metadata: %{}, + eligibility: :eligible, + retention: :durable + }) + end + + defp import_creators(candidate, artists, work_id, artwork_record, client, sources, run) do + by_slug = Map.new(candidate["creators"] || [], &{&1["artsy_artist_slug"], &1}) + + Enum.reduce(artists, {%{resolved: 0, links: 0, missing: 0}, client}, fn artist, + {stats, client} -> + case by_slug[artist["slug"]] do + %{"qid" => qid} = creator when is_binary(qid) -> + {person_id, _record} = ensure_wikidata_creator!(creator, sources["wikidata"], run.id) + + {:ok, artist_record} = + Sources.upsert_record(sources["artsy"], %{ + external_id: "artist:#{artist["id"]}", + url: artist["permalink"], + raw: %{ + "artist" => artist, + "wikidata_qid" => qid, + "wikidata_p2042" => creator["artsy_artist_slug"], + "retention" => %{"remove_on_provider_termination" => true} + }, + import_run_id: run.id + }) + + {:ok, entry} = + Entry.new(%{ + source_slug: "artsy", + source_id: sources["artsy"].id, + source_record_id: artist_record.id, + source_record_revision_id: artist_record.current_revision.id, + import_run_id: run.id, + object_kind: :entity, + entity_kind: :person, + stable_identifier: %{namespace: "artsy_artist_id", external_id: artist["id"]}, + identifiers: [ + %{namespace: "wikidata", external_id: qid}, + %{namespace: "artsy_artist_slug", external_id: artist["slug"]}, + %{namespace: "artsy_artist_id", external_id: artist["id"]} + ], + label: creator["name"] || artist["name"], + metadata: %{}, + eligibility: :eligible, + retention: :durable + }) + + resolution = SourceIdentity.resolve(entry) + + if resolution.object_id == person_id do + linked? = ensure_creator_link(work_id, person_id, sources["artsy"].id, artwork_record) + + {%{ + stats + | resolved: stats.resolved + 1, + links: stats.links + if(linked?, do: 1, else: 0) + }, client} + else + {%{stats | missing: stats.missing + 1}, client} + end + + _ -> + {%{stats | missing: stats.missing + 1}, client} + end + end) + end + + defp ensure_creator_link(work_id, person_id, source_id, artwork_record) do + origin_key = "artsy:creator:#{work_id}:#{person_id}" + + case DevilsDictionary.Repo.get_by(Assertion, source_id: source_id, origin_key: origin_key) do + nil -> + claim = + unwrap( + Claims.assert(work_id, "authored_by", person_id, %{ + source_id: source_id, + origin_key: origin_key, + method: "source_relationship", + metadata: %{"provider" => "artsy", "review" => "attributed_source_fact"} + }) + ) + + revision = Claims.current_revision(claim.id) + + unwrap( + Claims.add_evidence(revision.id, %{ + source_record_revision_id: artwork_record.current_revision.id, + evidence_role: :supports, + attribution_text: "Artsy artwork creator link" + }) + ) + + true + + _existing -> + false + end + end + + defp ensure_wikidata_work!(candidate, source, run_id) do + raw = wikidata_snapshot(candidate, :artwork) + + {:ok, record} = + Sources.upsert_record(source, %{ + external_id: candidate["qid"], + url: "https://www.wikidata.org/wiki/#{candidate["qid"]}", + raw: raw, + import_run_id: run_id + }) + + {:ok, _} = Materializer.run(record, Wikidata, run_id: run_id) + {Registry.by_external_id("wikidata", candidate["qid"]), record} + end + + defp ensure_wikidata_creator!(creator, source, run_id) do + raw = wikidata_snapshot(creator, :person) + + {:ok, record} = + Sources.upsert_record(source, %{ + external_id: creator["qid"], + url: "https://www.wikidata.org/wiki/#{creator["qid"]}", + raw: raw, + import_run_id: run_id + }) + + {:ok, _} = Materializer.run(record, Wikidata, run_id: run_id) + {Registry.by_external_id("wikidata", creator["qid"]), record} + end + + defp wikidata_snapshot(value, kind) do + claims = + case kind do + :artwork -> + %{ + "P31" => [entity_statement("Q3305213")], + "P11005" => [string_statement(value["artsy_artwork_slug"])], + "P170" => Enum.map(value["creators"] || [], &entity_statement(&1["qid"])) + } + + :person -> + %{ + "P31" => [entity_statement("Q5")], + "P2042" => [string_statement(value["artsy_artist_slug"])] + } + end + + %{ + "_artwork_seed_version" => 1, + "id" => value["qid"], + "labels" => %{"en" => %{"language" => "en", "value" => value["title"] || value["name"]}}, + "descriptions" => description(value["description"]), + "aliases" => %{}, + "sitelinks" => sitelink(value["wikipedia_title"]), + "claims" => claims + } + end + + defp entity_statement(nil), do: %{} + + defp entity_statement(qid) do + %{ + "type" => "statement", + "rank" => "normal", + "mainsnak" => %{"datavalue" => %{"value" => %{"id" => qid}, "type" => "wikibase-entityid"}} + } + end + + defp string_statement(value) do + %{ + "type" => "statement", + "rank" => "normal", + "mainsnak" => %{"datavalue" => %{"value" => value, "type" => "string"}} + } + end + + defp description(nil), do: %{} + defp description(value), do: %{"en" => %{"language" => "en", "value" => value}} + defp sitelink(nil), do: %{} + defp sitelink(title), do: %{"enwiki" => %{"site" => "enwiki", "title" => title}} + + defp eligible_candidate?(candidate) do + candidate["kind"] == "painting" and Regex.match?(~r/\AQ[1-9]\d*\z/, candidate["qid"] || "") and + Regex.match?(~r/\A[a-z0-9][a-z0-9-]{1,254}\z/, candidate["artsy_artwork_slug"] || "") and + is_binary(candidate["title"]) and String.trim(candidate["title"]) != "" + end + + defp accumulate(summary, outcome, index) do + summary = %{summary | processed: summary.processed + 1, next_index: index + 1} + + summary = + case outcome.status do + :created -> %{summary | created: summary.created + 1} + :matched -> %{summary | matched: summary.matched + 1} + :conflict -> %{summary | conflicts: summary.conflicts + 1} + :unavailable -> %{summary | unavailable: summary.unavailable + 1} + :quota_exhausted -> summary + :skipped -> %{summary | skipped: summary.skipped + 1} + end + + summary + |> Map.update!(:reproductions, &(&1 + if(outcome[:reproduction], do: 1, else: 0))) + |> Map.update!(:creators_resolved, &(&1 + (outcome[:creator_count] || 0))) + |> Map.update!(:creator_links, &(&1 + (outcome[:creator_links] || 0))) + |> Map.update!(:missing_links, &(&1 + (outcome[:missing_creators] || 0))) + |> Map.update!(:genes, &(&1 + (outcome[:gene_count] || 0))) + end + + defp maybe_save(manifest, nil), do: manifest + defp maybe_save(manifest, path), do: Manifest.save!(manifest, path) + defp bounded(value, min, max) when is_integer(value), do: value |> max(min) |> min(max) + defp stringify(map), do: Map.new(map, fn {key, value} -> {to_string(key), value} end) + defp timestamp, do: DateTime.utc_now() |> DateTime.truncate(:second) |> DateTime.to_iso8601() + defp unwrap({:ok, value}), do: value + defp unwrap({:error, reason}), do: DevilsDictionary.Repo.rollback(reason) +end diff --git a/lib/devils_dictionary/artworks/wikidata_candidates.ex b/lib/devils_dictionary/artworks/wikidata_candidates.ex new file mode 100644 index 00000000..8acc1b77 --- /dev/null +++ b/lib/devils_dictionary/artworks/wikidata_candidates.ex @@ -0,0 +1,225 @@ +defmodule DevilsDictionary.Artworks.WikidataCandidates do + @moduledoc "Bounded local-first discovery of painting candidates carrying Wikidata P11005." + + import Ecto.Query + + alias DevilsDictionary.Registry.{Entity, ExternalIdentifier, WorkDetails} + alias DevilsDictionary.Repo + + @endpoint "https://query.wikidata.org/sparql" + @default_limit 50 + @max_limit 5_000 + @default_page_size 50 + @max_requests 100 + + @doc "Returns local candidates first, then bounded paginated Wikidata SPARQL candidates." + def discover(opts \\ []) do + limit = bounded(opts[:limit] || @default_limit, 1, @max_limit, :limit) + request_limit = bounded(opts[:request_limit] || 10, 1, @max_requests, :request_limit) + page_size = bounded(opts[:page_size] || @default_page_size, 1, 200, :page_size) + offset = bounded(opts[:offset] || 0, 0, 100_000, :offset) + remote? = Keyword.get(opts, :remote, true) + request_fun = opts[:request_fun] || (&default_request/1) + + local = local_candidates(limit) + remaining = max(limit - length(local), 0) + + {remote, stats} = + if remote? and remaining > 0 do + remote_candidates(remaining, offset, page_size, request_limit, request_fun) + else + {[], %{requests: 0, next_offset: offset, truncated: remaining > 0}} + end + + candidates = + (local ++ remote) + |> Enum.uniq_by(&{&1["qid"], &1["artsy_artwork_slug"]}) + |> Enum.take(limit) + + {:ok, candidates, + Map.merge(stats, %{ + local: length(local), + remote: length(candidates) - length(local), + discovered: length(candidates), + limit: limit, + kinds: ["painting"] + })} + end + + defp local_candidates(limit) do + Repo.all( + from entity in Entity, + join: details in WorkDetails, + on: details.entity_id == entity.object_id and details.work_kind == "artwork", + join: artsy in ExternalIdentifier, + on: + artsy.object_id == entity.object_id and artsy.namespace == "artsy_artwork_slug" and + artsy.status == :verified, + join: qid in ExternalIdentifier, + on: + qid.object_id == entity.object_id and qid.namespace == "wikidata" and + qid.status == :verified, + order_by: [asc: entity.object_id], + limit: ^limit, + select: %{ + "qid" => qid.external_id, + "artsy_artwork_slug" => artsy.external_id, + "title" => entity.preferred_label, + "description" => entity.description, + "wikipedia_title" => entity.metadata["wikipedia_title"], + "kind" => "painting", + "selection_reason" => "existing_local_wikidata_p11005", + "creators" => [] + } + ) + end + + defp remote_candidates(limit, offset, page_size, request_limit, request_fun) do + do_remote([], limit, offset, page_size, request_limit, 0, request_fun) + end + + defp do_remote(rows, limit, offset, _page_size, request_limit, requests, _request_fun) + when length(rows) >= limit or requests >= request_limit do + {Enum.take(rows, limit), + %{requests: requests, next_offset: offset, truncated: length(rows) < limit}} + end + + defp do_remote(rows, limit, offset, page_size, request_limit, requests, request_fun) do + take = min(page_size, max(limit - length(rows), 1)) + + case request_fun.(request_options(take, offset)) do + {:ok, %Req.Response{status: 200, body: body}} -> + bindings = get_in(body, ["results", "bindings"]) || [] + page = normalize_bindings(bindings) + next_offset = offset + take + + if bindings == [] do + {Enum.take(rows, limit), + %{requests: requests + 1, next_offset: next_offset, truncated: false}} + else + do_remote( + dedupe(rows ++ page), + limit, + next_offset, + page_size, + request_limit, + requests + 1, + request_fun + ) + end + + {:ok, %Req.Response{status: status}} -> + {rows, + %{ + requests: requests + 1, + next_offset: offset, + truncated: true, + error: "wikidata_http_#{status}" + }} + + {:error, _reason} -> + {rows, + %{ + requests: requests + 1, + next_offset: offset, + truncated: true, + error: "wikidata_unavailable" + }} + end + end + + defp request_options(limit, offset) do + [ + method: :get, + url: @endpoint, + params: [query: sparql(limit, offset), format: "json"], + headers: [ + {"accept", "application/sparql-results+json"}, + {"user-agent", Application.fetch_env!(:devils_dictionary, :user_agent)} + ], + receive_timeout: 15_000, + retry: :transient, + max_retries: 2 + ] + end + + defp default_request(options), do: Req.request(options) + + defp sparql(limit, offset) do + """ + SELECT ?item ?itemLabel ?itemDescription ?artsy ?article ?creator ?creatorLabel ?creatorArtsy WHERE { + ?item wdt:P31 wd:Q3305213 ; wdt:P11005 ?artsy . + OPTIONAL { ?article schema:about ?item ; schema:isPartOf . } + OPTIONAL { + ?item wdt:P170 ?creator . + OPTIONAL { ?creator wdt:P2042 ?creatorArtsy . } + } + SERVICE wikibase:label { bd:serviceParam wikibase:language "en,mul". } + } + ORDER BY DESC(BOUND(?article)) ?item ?creator + LIMIT #{limit} + OFFSET #{offset} + """ + end + + defp normalize_bindings(bindings) do + bindings + |> Enum.group_by(&qid(value(&1, "item"))) + |> Enum.reject(fn {qid, _} -> is_nil(qid) end) + |> Enum.map(fn {qid, rows} -> + first = hd(rows) + + %{ + "qid" => qid, + "artsy_artwork_slug" => value(first, "artsy"), + "title" => value(first, "itemLabel"), + "description" => value(first, "itemDescription"), + "wikipedia_title" => wikipedia_title(value(first, "article")), + "kind" => "painting", + "selection_reason" => + if(value(first, "article"), + do: "wikidata_p11005_with_english_wikipedia", + else: "wikidata_p11005_painting" + ), + "creators" => + rows + |> Enum.map(fn row -> + %{ + "qid" => qid(value(row, "creator")), + "name" => value(row, "creatorLabel"), + "artsy_artist_slug" => value(row, "creatorArtsy") + } + end) + |> Enum.reject(&is_nil(&1["qid"])) + |> Enum.uniq_by(& &1["qid"]) + } + end) + |> Enum.sort_by(& &1["qid"]) + end + + defp dedupe(rows), do: Enum.uniq_by(rows, &{&1["qid"], &1["artsy_artwork_slug"]}) + defp value(row, key), do: get_in(row, [key, "value"]) + + defp qid("http://www.wikidata.org/entity/" <> qid), do: if(valid_qid?(qid), do: qid) + defp qid("https://www.wikidata.org/entity/" <> qid), do: if(valid_qid?(qid), do: qid) + defp qid(_), do: nil + defp valid_qid?(qid), do: Regex.match?(~r/\AQ[1-9]\d*\z/, qid) + + defp wikipedia_title(nil), do: nil + + defp wikipedia_title(url) do + case URI.parse(url) do + %URI{scheme: "https", host: "en.wikipedia.org", path: "/wiki/" <> title} -> + title |> URI.decode() |> String.replace("_", " ") + + _ -> + nil + end + end + + defp bounded(value, min, max, _name) when is_integer(value) and value >= min and value <= max, + do: value + + defp bounded(_value, _min, _max, name), + do: raise(ArgumentError, "#{name} is outside the supported bound") +end diff --git a/lib/devils_dictionary/discovery/providers.ex b/lib/devils_dictionary/discovery/providers.ex index 70de9f87..1b465d09 100644 --- a/lib/devils_dictionary/discovery/providers.ex +++ b/lib/devils_dictionary/discovery/providers.ex @@ -1,9 +1,9 @@ defmodule DevilsDictionary.Discovery.Providers do @moduledoc "The configured provider registry used by pages, workers and `mix dd.discovery`." - alias DevilsDictionary.Discovery.Providers.{CineGraph, Giphy} + alias DevilsDictionary.Discovery.Providers.{Artsy, CineGraph, Giphy} - @default [CineGraph, Giphy] + @default [Artsy, CineGraph, Giphy] def all do Application.get_env(:devils_dictionary, :discovery_providers, @default) diff --git a/lib/devils_dictionary/discovery/providers/artsy.ex b/lib/devils_dictionary/discovery/providers/artsy.ex new file mode 100644 index 00000000..ce493edd --- /dev/null +++ b/lib/devils_dictionary/discovery/providers/artsy.ex @@ -0,0 +1,70 @@ +defmodule DevilsDictionary.Discovery.Providers.Artsy do + @moduledoc "Artsy source registration and explicit server-side artwork discovery capabilities." + + @behaviour DevilsDictionary.Discovery.Provider + + @impl true + def slug, do: "artsy" + + @impl true + def adapter_version, do: "artsy.rest.v1" + + @impl true + def source_attrs do + %{ + slug: slug(), + name: "Artsy", + tier: :middle, + kind: :media_provider, + access: :api, + era_year: 2026, + license: "Artsy Public API Terms; removable cache, no permanent archive grant", + license_url: "https://developers.artsy.net/v2/terms", + homepage: "https://www.artsy.net/", + url_template: "https://www.artsy.net/artwork/{external_id}", + attribution: "Artwork discovery and source metadata: Artsy", + config: %{ + "mode" => "explicit_server_search_and_selected_enrichment", + "retention" => "disposable cache unless independently supported", + "image_storage" => "references only; no downloads", + "gene_traversal" => "disabled_pending_control_verification", + "retirement_notice" => true + } + } + end + + @impl true + def capabilities do + %{ + background: false, + transport: :server, + persistence: :persistent, + pagination: :offset, + operations: ["artwork_search"], + content_types: [:artwork] + } + end + + @impl true + def enabled? do + config = Application.get_env(:devils_dictionary, :artsy, []) + + config[:enabled] != false and present?(config[:client_id]) and + present?(config[:client_secret]) + end + + @impl true + def validate_mapping("artwork_search", %{ + "query" => query, + "language" => language, + "match_type" => match_type + }) + when is_binary(query) and byte_size(query) > 0 and byte_size(query) <= 200 and + is_binary(language) and match_type in ["exact", "broader", "related"], + do: :ok + + def validate_mapping(_operation, _parameters), do: {:error, :unsupported_filter} + + defp present?(value) when is_binary(value), do: String.trim(value) != "" + defp present?(_), do: false +end diff --git a/lib/devils_dictionary_web/components/artwork.ex b/lib/devils_dictionary_web/components/artwork.ex new file mode 100644 index 00000000..b90d9c7a --- /dev/null +++ b/lib/devils_dictionary_web/components/artwork.ex @@ -0,0 +1,100 @@ +defmodule DevilsDictionaryWeb.Artwork do + @moduledoc "Reusable, identity-addressed artwork entries and meaning candidates." + + use DevilsDictionaryWeb, :html + + alias DevilsDictionary.Claims.Connection + + attr :artwork, :map, required: true + attr :candidate, :map, default: nil + attr :connect, :boolean, default: false + attr :id, :string, required: true + + def card(assigns) do + ~H""" +
+ <.link + navigate={entity_path(@artwork)} + class="aspect-[4/5] overflow-hidden rounded-sm bg-mist-950/5 transition-transform duration-200 group-hover:-translate-y-0.5 dark:bg-white/5" + aria-label={"Open #{@artwork.title}"} + > + + + <.icon name="hero-photo" class="size-6 stroke-current" /> + + + +
+ <.link + navigate={entity_path(@artwork)} + class="font-display text-xl text-balance text-mist-950 underline-offset-4 group-hover:underline dark:text-white" + > + {@artwork.title} + +

+ by + + 0}>, <.link + navigate={creator_path(creator)} + class="hover:underline" + >{creator.label} + +

+

+ {@artwork.description} +

+ +
+

+ Candidate for “{@candidate.meaning}” + ({@candidate.language}) +

+

+ Direct Artsy gene “{@candidate.match_reason.gene_name}” · not yet reviewed +

+
+ +
+ <.link navigate={entity_path(@artwork)} class="font-medium underline underline-offset-4">Open work + <.link + :if={@connect} + navigate={~p"/connect?subject=#{@artwork.object_id}"} + class="font-medium underline underline-offset-4" + > + Connect to a meaning + + Artsy source ↗ +
+
+
+ """ + end + + defp entity_path(artwork), + do: ~p"/entities/#{artwork.object_id}/#{Connection.slugify(artwork.title)}" + + defp creator_path(creator), + do: ~p"/entities/#{creator.object_id}/#{Connection.slugify(creator.label)}" +end diff --git a/lib/devils_dictionary_web/components/layouts.ex b/lib/devils_dictionary_web/components/layouts.ex index 91e95dfe..342427f6 100644 --- a/lib/devils_dictionary_web/components/layouts.ex +++ b/lib/devils_dictionary_web/components/layouts.ex @@ -52,6 +52,9 @@ defmodule DevilsDictionaryWeb.Layouts do <.footer> <:links> + <.footer_category title="Browse"> + <.footer_link navigate={~p"/artworks"}>Artworks + <.footer_category title="Sources"> <.footer_link :for={source <- sources()} navigate={~p"/sources/#{source.slug}"}> {source.name} diff --git a/lib/devils_dictionary_web/live/artwork_live.ex b/lib/devils_dictionary_web/live/artwork_live.ex new file mode 100644 index 00000000..304fdcba --- /dev/null +++ b/lib/devils_dictionary_web/live/artwork_live.ex @@ -0,0 +1,252 @@ +defmodule DevilsDictionaryWeb.ArtworkLive do + @moduledoc "Local artwork catalog with an explicitly transient Artsy availability check." + + use DevilsDictionaryWeb, :live_view + + alias DevilsDictionary.Artsy.Client + alias DevilsDictionary.Artworks + alias DevilsDictionary.Claims.Contributions + alias DevilsDictionary.Registry + alias DevilsDictionaryWeb.Artwork + + on_mount {DevilsDictionaryWeb.UserAuth, :mount_current_scope} + + @impl true + def mount(_params, _session, socket) do + configured? = Client.new() |> Client.configured?() + + {:ok, + socket + |> assign( + page_title: "artworks", + search_form: to_form(%{"q" => ""}, as: :search), + artsy_form: to_form(%{"q" => ""}, as: :artsy), + artsy_configured: configured?, + artsy_requests_used: 0, + artsy_status: :idle, + artsy_items: [], + artsy_query: "", + contributor: Contributions.internal_contributor?(socket.assigns[:current_scope]) + ) + |> stream(:artworks, Artworks.search(""), dom_id: &"artwork-#{&1.object_id}")} + end + + @impl true + def handle_event("search-local", %{"search" => %{"q" => query}}, socket) do + {:noreply, + socket + |> assign(:search_form, to_form(%{"q" => query}, as: :search)) + |> stream(:artworks, Artworks.search(query), reset: true)} + end + + def handle_event("search-artsy", %{"artsy" => %{"q" => query}}, socket) do + query = String.trim(query) + + cond do + not socket.assigns.artsy_configured -> + {:noreply, assign(socket, artsy_status: :disabled, artsy_items: [])} + + query == "" -> + {:noreply, assign(socket, artsy_status: :idle, artsy_items: [], artsy_query: "")} + + socket.assigns.artsy_requests_used >= 30 -> + {:noreply, assign(socket, artsy_status: :quota, artsy_items: [])} + + true -> + # The credential-bearing client exists only inside this server task; + # it is never retained in LiveView assigns or rendered state. + client = Client.new(request_limit: 30 - socket.assigns.artsy_requests_used) + + {:noreply, + socket + |> assign( + artsy_form: to_form(%{"q" => query}, as: :artsy), + artsy_status: :loading, + artsy_query: query, + artsy_items: [] + ) + |> start_async(:artsy_search, fn -> Client.search_artworks(client, query, size: 8) end)} + end + end + + @impl true + def handle_async(:artsy_search, {:ok, {:ok, result, client}}, socket) do + items = Enum.map(result.items, &decorate_result/1) + + {:noreply, + assign(socket, + artsy_requests_used: socket.assigns.artsy_requests_used + client.request_count, + artsy_status: if(items == [], do: :empty, else: :ready), + artsy_items: items + )} + end + + def handle_async(:artsy_search, {:ok, {:error, failure, client}}, socket) do + status = if failure.code == "request_limit", do: :quota, else: :failed + + {:noreply, + assign(socket, + artsy_requests_used: socket.assigns.artsy_requests_used + client.request_count, + artsy_status: status, + artsy_items: [] + )} + end + + def handle_async(:artsy_search, {:exit, _reason}, socket), + do: {:noreply, assign(socket, artsy_status: :failed, artsy_items: [])} + + defp decorate_result(%{status: :available, artwork: %{"slug" => slug}} = item) do + Map.put(item, :local, local_artwork(slug)) + end + + defp decorate_result(item), do: Map.put(item, :local, nil) + + defp local_artwork(slug) do + case Registry.by_external_id("artsy_artwork_slug", slug) do + nil -> nil + object_id -> Artworks.get(object_id) + end + end + + @impl true + def render(assigns) do + ~H""" + + <.container class="py-10"> +
+ <.eyebrow>the collection + <.heading>Artworks, connected by identity + <.text class="mt-3"> + Search the works already held in Dictionary. Each title and creator opens the same + reusable identity used by meanings, evidence, and review—not a copied search result. + +
+ + <.form + for={@search_form} + id="artwork-search" + phx-change="search-local" + class="mt-8 max-w-xl" + > + <.input + field={@search_form[:q]} + type="search" + label="Search saved artworks or creators" + placeholder="Goya, The Third of May…" + phx-debounce="250" + /> + + +
+
+ + +
+
+ +
+ <.eyebrow>source availability check + <.subheading class="mt-2">Look in Artsy without adding a claim + <.text class="mt-2"> + This bounded server-side lookup is temporary discovery. A result is neither a saved + Dictionary work nor evidence for a meaning. Unavailable records are shown honestly. + + + <.form + for={@artsy_form} + id="artsy-search" + phx-submit="search-artsy" + class="mt-5 flex max-w-xl items-end gap-3" + > +
+ <.input + field={@artsy_form[:q]} + type="search" + label="Check Artsy" + placeholder="The Third of May" + disabled={!@artsy_configured} + /> +
+ <.button + id="artsy-search-submit" + type="submit" + disabled={!@artsy_configured || @artsy_status == :loading} + > + {if @artsy_status == :loading, do: "Checking…", else: "Check source"} + + + +

+ Artsy is not configured on this server. The saved local collection remains available. +

+

+ {if @artsy_status == :quota, + do: "This lookup reached its request bound.", + else: "Artsy is temporarily unavailable."} +

+

+ No artwork results were returned for “{@artsy_query}”. +

+ +
    +
  • + <%= if item.local do %> + <.link + navigate={ + ~p"/entities/#{item.local.object_id}/#{DevilsDictionary.Claims.Connection.slugify(item.local.title)}" + } + class="font-medium underline underline-offset-4" + > + {item.local.title} + +

    Already saved · exact P11005 match

    + <% else %> +

    {item.search["title"] || "Untitled result"}

    +

    {artsy_result_label(item.status)}

    + Open source ↗ + <% end %> +
  • +
+
+ +
+ """ + end + + defp artsy_result_label(:available), + do: "Available at source · not saved and not a meaning claim" + + defp artsy_result_label(:unavailable), do: "Search hit, but the artwork endpoint is unavailable" + defp artsy_result_label(:unsupported_type), do: "Unsupported non-artwork result" + defp artsy_result_label(:invalid_endpoint), do: "Ignored unsafe provider endpoint" + defp artsy_result_label(_), do: "Source lookup failed for this result" +end diff --git a/lib/devils_dictionary_web/live/connection_live.ex b/lib/devils_dictionary_web/live/connection_live.ex index fbe786b1..7da131b5 100644 --- a/lib/devils_dictionary_web/live/connection_live.ex +++ b/lib/devils_dictionary_web/live/connection_live.ex @@ -166,8 +166,24 @@ defmodule DevilsDictionaryWeb.ConnectionLive do end end - def handle_params(_params, _uri, socket) do - {:noreply, assign(socket, page_title: "propose a connection")} + def handle_params(params, _uri, socket) do + subject = preselected_subject(params["subject"]) + + {:noreply, + assign(socket, + page_title: "propose a connection", + subject: subject, + predicates: predicates_for(subject) + )} + end + + defp preselected_subject(nil), do: nil + + defp preselected_subject(value) do + case Integer.parse(value) do + {id, ""} when id > 0 -> Connection.endpoint(id) + _ -> nil + end end defp revision_number(nil), do: {:ok, nil} diff --git a/lib/devils_dictionary_web/live/word_live.ex b/lib/devils_dictionary_web/live/word_live.ex index 843922af..fa2338b1 100644 --- a/lib/devils_dictionary_web/live/word_live.ex +++ b/lib/devils_dictionary_web/live/word_live.ex @@ -35,9 +35,10 @@ defmodule DevilsDictionaryWeb.WordLive do alias DevilsDictionary.Demo, as: Samples alias DevilsDictionary.Discovery alias DevilsDictionary.Discovery.Providers + alias DevilsDictionary.Artworks alias DevilsDictionary.Lexicon alias DevilsDictionary.Lexicon.WordPage - alias DevilsDictionaryWeb.{Culture, Demo, Provenance, Thing, Word} + alias DevilsDictionaryWeb.{Artwork, Culture, Demo, Provenance, Thing, Word} @trail_cap 12 @suggestions 5 @@ -53,6 +54,7 @@ defmodule DevilsDictionaryWeb.WordLive do evidence: [], object_id: nil, choices: [], + artwork_candidates: [], discovery_target: nil, cultures: %{} )} @@ -126,6 +128,7 @@ defmodule DevilsDictionaryWeb.WordLive do |> assign(:page_title, title(page, slug)) |> assign(:card_sources, card_sources) |> assign(:suggestions, suggestions(page, slug)) + |> assign(:artwork_candidates, artwork_candidates(page)) |> assign(:choices, choices(slug, socket.assigns.object_id)) prepare_discovery(socket, page, demo) @@ -326,6 +329,10 @@ defmodule DevilsDictionaryWeb.WordLive do defp suggestions(_page, _slug), do: [] + defp artwork_candidates(%{headword: %{lexemes: lexemes}}) do + Artworks.suggestions(Enum.map(lexemes, & &1.id)) + end + # The trail is user input arriving in a URL, so it is parsed rather than # trusted: slugs only, deduplicated, and the most recent twelve. #71 §10 # keeps it here instead of in socket state so a walk survives a reload and @@ -383,6 +390,32 @@ defmodule DevilsDictionaryWeb.WordLive do +
+
+
+ <.eyebrow>artwork candidates + <.subheading class="mt-1">Works to consider for an exact meaning +
+ <.a navigate={~p"/artworks"}>Browse saved artworks +
+

+ Direct source tags produced these candidates. They are not accepted interpretations; + a contributor must connect an exact sense and reviewers decide the claim. +

+
+ +
+
+ Mix.shell().info("Artsy withdrawal complete: " <> inspect(summary)) + {:error, error} -> Mix.raise("Artsy withdrawal failed: #{inspect(error)}") + end + end +end diff --git a/lib/mix/tasks/dd.artworks.seed.ex b/lib/mix/tasks/dd.artworks.seed.ex new file mode 100644 index 00000000..aae8757d --- /dev/null +++ b/lib/mix/tasks/dd.artworks.seed.ex @@ -0,0 +1,93 @@ +defmodule Mix.Tasks.Dd.Artworks.Seed do + @shortdoc "Builds or runs a bounded Wikidata/Artsy artwork seed manifest" + + @moduledoc """ + Builds a credential-free, checksummed manifest from exact Wikidata P11005 + identifiers, or imports an existing manifest through the shared source + identity resolver. + + mix dd.artworks.seed --manifest priv/artworks/manifests/pilot-v1.json --build --dry-run + mix dd.artworks.seed --manifest priv/artworks/manifests/pilot-v1.json --resume + + `--dry-run` never writes the database. `--build` discovers candidates and + writes only the manifest; add `--import` to build and import in one command. + All discovery and provider calls are bounded. Reruns converge on exact + Wikidata, P11005/P2042 and opaque Artsy identifiers; titles never identify. + """ + + use Mix.Task + + alias DevilsDictionary.Artworks.{Manifest, Seeder, WikidataCandidates} + + @switches [ + manifest: :string, + build: :boolean, + import: :boolean, + dry_run: :boolean, + resume: :boolean, + local_only: :boolean, + candidate_limit: :integer, + request_limit: :integer, + discovery_request_limit: :integer, + record_limit: :integer, + batch_size: :integer, + offset: :integer + ] + + @impl Mix.Task + def run(args) do + Mix.Task.run("app.start") + {opts, rest, invalid} = OptionParser.parse(args, strict: @switches) + + if rest != [] or invalid != [] or is_nil(opts[:manifest]) do + Mix.raise("invalid arguments; --manifest is required (run `mix help dd.artworks.seed`)") + end + + path = Path.expand(opts[:manifest]) + manifest = if opts[:build], do: build(path, opts), else: Manifest.load!(path) + + if opts[:build] == true and opts[:import] != true do + print_manifest(manifest, path) + else + {manifest, summary} = + Seeder.run(manifest, + dry_run: opts[:dry_run] || false, + resume: opts[:resume] || false, + manifest_path: if(opts[:dry_run], do: nil, else: path), + request_limit: opts[:request_limit] || 500, + record_limit: opts[:record_limit] || 500, + batch_size: opts[:batch_size] || 25 + ) + + print_manifest(manifest, path) + Mix.shell().info("Artwork seed summary: " <> inspect(summary, pretty: true)) + end + rescue + error in [ArgumentError, File.Error] -> Mix.raise(Exception.message(error)) + end + + defp build(path, opts) do + {:ok, candidates, stats} = + WikidataCandidates.discover( + limit: opts[:candidate_limit] || 50, + request_limit: opts[:discovery_request_limit] || 10, + offset: opts[:offset] || 0, + remote: not (opts[:local_only] || false) + ) + + manifest = + Manifest.new(candidates, %{ + "discovery" => stats, + "source" => "Wikidata P11005/P2042", + "selection_reason" => "painting with exact Artsy artwork identifier" + }) + + Manifest.save!(manifest, path) + end + + defp print_manifest(manifest, path) do + Mix.shell().info( + "Artwork manifest #{path}: #{length(manifest["candidates"])} candidates, checksum #{manifest["checksum"]}" + ) + end +end diff --git a/priv/artworks/manifests/pilot-v1.json b/priv/artworks/manifests/pilot-v1.json new file mode 100644 index 00000000..43d84425 --- /dev/null +++ b/priv/artworks/manifests/pilot-v1.json @@ -0,0 +1,864 @@ +{ + "candidates": [ + { + "artsy_artwork_slug": "anna-rimbault-borrel-valentine-de-milan-and-odette-de-champs-divers-from-the-alexander-dumas-novel-isabelle-de-baviere", + "creators": [ + { + "artsy_artist_slug": "anna-rimbault-borrel", + "name": "Anna Rimbaut-Borrel", + "qid": "Q86617778" + } + ], + "description": "painting by Anna Rimbaut-Borrel", + "import": { + "object_id": 3733926, + "reason": "artsy_artwork_not_found", + "status": "unavailable", + "updated_at": "2026-09-12T18:23:13Z" + }, + "kind": "painting", + "qid": "Q106249870", + "selection_reason": "wikidata_p11005_painting", + "title": "Valentine of Milan and Odette de Champdivers", + "wikipedia_title": null + }, + { + "artsy_artwork_slug": "paul-klee-tor-im-garten-gate-in-the-garden", + "creators": [ + { + "artsy_artist_slug": "paul-klee", + "name": "Paul Klee", + "qid": "Q44007" + } + ], + "description": "painting by Paul Klee", + "import": { + "object_id": 3733927, + "reason": "artsy_artwork_not_found", + "status": "unavailable", + "updated_at": "2026-09-12T18:23:14Z" + }, + "kind": "painting", + "qid": "Q116689343", + "selection_reason": "wikidata_p11005_painting", + "title": "Gate in the Garden", + "wikipedia_title": null + }, + { + "artsy_artwork_slug": "raphael-portrait-of-pope-julius-ii", + "creators": [ + { + "artsy_artist_slug": "raphael", + "name": "Raphael", + "qid": "Q5597" + } + ], + "description": "painting by Raphael in the National Gallery, London", + "import": { + "object_id": 3733928, + "reason": "artsy_artwork_not_found", + "status": "unavailable", + "updated_at": "2026-09-12T18:23:15Z" + }, + "kind": "painting", + "qid": "Q11918714", + "selection_reason": "wikidata_p11005_painting", + "title": "Portrait of Pope Julius II", + "wikipedia_title": null + }, + { + "artsy_artwork_slug": "titian-bacchus-and-ariadne-1", + "creators": [ + { + "artsy_artist_slug": "titian", + "name": "Titian", + "qid": "Q47551" + } + ], + "description": "painting by Titian", + "import": { + "object_id": 3733929, + "reason": "artsy_artwork_not_found", + "status": "unavailable", + "updated_at": "2026-09-12T18:23:16Z" + }, + "kind": "painting", + "qid": "Q1206860", + "selection_reason": "wikidata_p11005_with_english_wikipedia", + "title": "Bacchus and Ariadne", + "wikipedia_title": "Bacchus and Ariadne" + }, + { + "artsy_artwork_slug": "titian-cupid-with-the-wheel-of-fortune", + "creators": [ + { + "artsy_artist_slug": "titian", + "name": "Titian", + "qid": "Q47551" + } + ], + "description": "painting by Titian", + "import": { + "artist_failure": null, + "artsy_artwork_id": "515d39335eeb1c524c0039df", + "artsy_artwork_slug": "titian-cupid-with-the-wheel-of-fortune", + "creator_count": 1, + "creator_links": 0, + "gene_count": 13, + "gene_failure": null, + "missing_creators": 0, + "object_id": 3733930, + "redirects": 1, + "source_record_id": 6984025, + "status": "matched", + "updated_at": "2026-09-12T18:23:18Z" + }, + "kind": "painting", + "qid": "Q12115903", + "selection_reason": "wikidata_p11005_painting", + "title": "Cupid with the Wheel of Time", + "wikipedia_title": null + }, + { + "artsy_artwork_slug": "titian-ranuccio-farnese-1", + "creators": [ + { + "artsy_artist_slug": "titian", + "name": "Titian", + "qid": "Q47551" + } + ], + "description": "painting by Titian", + "import": { + "status": "pending" + }, + "kind": "painting", + "qid": "Q12146958", + "selection_reason": "wikidata_p11005_with_english_wikipedia", + "title": "Portrait of Ranuccio Farnese", + "wikipedia_title": "Portrait of Ranuccio Farnese" + }, + { + "artsy_artwork_slug": "ferdinand-hodler-the-disappointed-souls-les-ames-decues", + "creators": [ + { + "artsy_artist_slug": "ferdinand-hodler", + "name": "Ferdinand Hodler", + "qid": "Q214564" + } + ], + "description": "1892 painting by Ferdinand Hodler", + "import": { + "status": "pending" + }, + "kind": "painting", + "qid": "Q122978100", + "selection_reason": "wikidata_p11005_painting", + "title": "Les Âmes Déçues", + "wikipedia_title": null + }, + { + "artsy_artwork_slug": "amable-paul-coutan-portrait-of-francisco-pizarro", + "creators": [ + { + "artsy_artist_slug": "amable-paul-coutan", + "name": "Amable-Paul Coutan", + "qid": "Q4738989" + } + ], + "description": "painting by Amable-Paul Coutan", + "import": { + "status": "pending" + }, + "kind": "painting", + "qid": "Q123239773", + "selection_reason": "wikidata_p11005_painting", + "title": "Francisco Pizarro, conqueror (1475–1541)", + "wikipedia_title": null + }, + { + "artsy_artwork_slug": "tiziano-vecellio-possible-portrait-of-jan-van-haanen", + "creators": [ + { + "artsy_artist_slug": "titian", + "name": "Titian", + "qid": "Q47551" + } + ], + "description": "painting by Titian", + "import": { + "status": "pending" + }, + "kind": "painting", + "qid": "Q123905131", + "selection_reason": "wikidata_p11005_painting", + "title": "Possible portrait of Giovanni d'Anna", + "wikipedia_title": null + }, + { + "artsy_artwork_slug": "jacopo-palma-il-giovane-self-portrait", + "creators": [ + { + "artsy_artist_slug": "jacopo-palma-il-giovane", + "name": "Palma il Giovane", + "qid": "Q651795" + } + ], + "description": "painting by Palma il Giovane in the Pinacoteca di Brera", + "import": { + "status": "pending" + }, + "kind": "painting", + "qid": "Q126709512", + "selection_reason": "wikidata_p11005_painting", + "title": "Self-portrait", + "wikipedia_title": null + }, + { + "artsy_artwork_slug": "louise-abbema-sarah-bernhardt-hunting-with-hounds", + "creators": [ + { + "artsy_artist_slug": "louise-abbema", + "name": "Louise Abbéma", + "qid": "Q237911" + } + ], + "description": "painting by Louise Abbéma", + "import": { + "status": "pending" + }, + "kind": "painting", + "qid": "Q130206451", + "selection_reason": "wikidata_p11005_painting", + "title": "Sarah Bernhardt Hunting with Hounds", + "wikipedia_title": null + }, + { + "artsy_artwork_slug": "sandro-botticelli-the-birth-of-venus", + "creators": [ + { + "artsy_artist_slug": "sandro-botticelli", + "name": "Sandro Botticelli", + "qid": "Q5669" + } + ], + "description": "painting by Sandro Botticelli, Uffizi", + "import": { + "status": "pending" + }, + "kind": "painting", + "qid": "Q151047", + "selection_reason": "wikidata_p11005_with_english_wikipedia", + "title": "The Birth of Venus", + "wikipedia_title": "The Birth of Venus" + }, + { + "artsy_artwork_slug": "titian-portrait-of-a-lady-in-white", + "creators": [ + { + "artsy_artist_slug": "titian", + "name": "Titian", + "qid": "Q47551" + } + ], + "description": "painting by Titian", + "import": { + "status": "pending" + }, + "kind": "painting", + "qid": "Q17072626", + "selection_reason": "wikidata_p11005_with_english_wikipedia", + "title": "Portrait of a Lady in White", + "wikipedia_title": "Portrait of a Lady in White" + }, + { + "artsy_artwork_slug": "titian-assumption-of-the-virgin", + "creators": [ + { + "artsy_artist_slug": "titian", + "name": "Titian", + "qid": "Q47551" + } + ], + "description": "painting by Titian in Venice", + "import": { + "status": "pending" + }, + "kind": "painting", + "qid": "Q1899740", + "selection_reason": "wikidata_p11005_with_english_wikipedia", + "title": "Assumption of the Virgin", + "wikipedia_title": "Assumption of the Virgin (Titian)" + }, + { + "artsy_artwork_slug": "robert-rauschenberg-rebus", + "creators": [ + { + "artsy_artist_slug": "robert-rauschenberg", + "name": "Robert Rauschenberg", + "qid": "Q164358" + } + ], + "description": "painting by Robert Rauschenberg", + "import": { + "status": "pending" + }, + "kind": "painting", + "qid": "Q19887272", + "selection_reason": "wikidata_p11005_painting", + "title": "Rebus", + "wikipedia_title": null + }, + { + "artsy_artwork_slug": "robert-rauschenberg-canyon", + "creators": [ + { + "artsy_artist_slug": "robert-rauschenberg", + "name": "Robert Rauschenberg", + "qid": "Q164358" + } + ], + "description": "painting by Robert Rauschenberg", + "import": { + "status": "pending" + }, + "kind": "painting", + "qid": "Q19887473", + "selection_reason": "wikidata_p11005_with_english_wikipedia", + "title": "Canyon", + "wikipedia_title": "Canyon (Rauschenberg)" + }, + { + "artsy_artwork_slug": "henri-rousseau-tropical-forest-with-monkeys", + "creators": [ + { + "artsy_artist_slug": "henri-rousseau", + "name": "Henri Rousseau", + "qid": "Q156386" + } + ], + "description": "painting by Henri Rousseau", + "import": { + "status": "pending" + }, + "kind": "painting", + "qid": "Q20191494", + "selection_reason": "wikidata_p11005_painting", + "title": "Tropical Forest with Monkeys", + "wikipedia_title": null + }, + { + "artsy_artwork_slug": "roger-de-la-fresnaye-the-bathers", + "creators": [ + { + "artsy_artist_slug": "roger-de-la-fresnaye", + "name": "Roger de La Fresnaye", + "qid": "Q718882" + } + ], + "description": "painting by Roger de La Fresnaye", + "import": { + "status": "pending" + }, + "kind": "painting", + "qid": "Q20191624", + "selection_reason": "wikidata_p11005_painting", + "title": "The Bathers", + "wikipedia_title": null + }, + { + "artsy_artwork_slug": "sir-william-orpen-mrs-charles-s-carstairs", + "creators": [ + { + "artsy_artist_slug": "sir-william-orpen", + "name": "William Orpen", + "qid": "Q922483" + } + ], + "description": "painting by William Orpen", + "import": { + "status": "pending" + }, + "kind": "painting", + "qid": "Q20191834", + "selection_reason": "wikidata_p11005_painting", + "title": "Mrs. Charles S. Carstairs", + "wikipedia_title": null + }, + { + "artsy_artwork_slug": "giovanni-cariani-portrait-of-a-young-man-with-a-green-book", + "creators": [ + { + "artsy_artist_slug": "giovanni-cariani", + "name": "Giovanni Cariani", + "qid": "Q976391" + } + ], + "description": "painting attributed to Giovanni Cariani", + "import": { + "status": "pending" + }, + "kind": "painting", + "qid": "Q20202605", + "selection_reason": "wikidata_p11005_painting", + "title": "Portrait of a Young Man with a Green Book", + "wikipedia_title": null + }, + { + "artsy_artwork_slug": "diego-rivera-retrato-de-ramon-gomez-de-la-serna", + "creators": [ + { + "artsy_artist_slug": "diego-rivera", + "name": "Diego Rivera", + "qid": "Q171128" + } + ], + "description": "painting by Diego Rivera", + "import": { + "status": "pending" + }, + "kind": "painting", + "qid": "Q21606628", + "selection_reason": "wikidata_p11005_with_english_wikipedia", + "title": "Portrait of Ramón Gómez de la Serna", + "wikipedia_title": "Portrait of Ramón Gómez de la Serna" + }, + { + "artsy_artwork_slug": "titian-venus-rising-from-the-sea-a-venus-anadyomenea", + "creators": [ + { + "artsy_artist_slug": "titian", + "name": "Titian", + "qid": "Q47551" + } + ], + "description": "painting by Titian", + "import": { + "status": "pending" + }, + "kind": "painting", + "qid": "Q2466780", + "selection_reason": "wikidata_p11005_with_english_wikipedia", + "title": "Venus Anadyomene", + "wikipedia_title": "Venus Anadyomene (Titian)" + }, + { + "artsy_artwork_slug": "titian-diana-and-actaeon", + "creators": [ + { + "artsy_artist_slug": "titian", + "name": "Titian", + "qid": "Q47551" + } + ], + "description": "painting by Titian", + "import": { + "status": "pending" + }, + "kind": "painting", + "qid": "Q2528177", + "selection_reason": "wikidata_p11005_with_english_wikipedia", + "title": "Diana and Actaeon", + "wikipedia_title": "Diana and Actaeon (Titian)" + }, + { + "artsy_artwork_slug": "titian-pieta", + "creators": [ + { + "artsy_artist_slug": "jacopo-palma-il-giovane", + "name": "Palma il Giovane", + "qid": "Q651795" + }, + { + "artsy_artist_slug": "titian", + "name": "Titian", + "qid": "Q47551" + } + ], + "description": "painting by Titian, finished by Palma il Giovane", + "import": { + "status": "pending" + }, + "kind": "painting", + "qid": "Q2665840", + "selection_reason": "wikidata_p11005_with_english_wikipedia", + "title": "Pietà", + "wikipedia_title": "Pietà (Titian)" + }, + { + "artsy_artwork_slug": "gustav-klimt-portrait-of-hermine-gallia", + "creators": [ + { + "artsy_artist_slug": "gustav-klimt", + "name": "Gustav Klimt", + "qid": "Q34661" + } + ], + "description": "painting by Gustav Klimt", + "import": { + "status": "pending" + }, + "kind": "painting", + "qid": "Q26713913", + "selection_reason": "wikidata_p11005_painting", + "title": "Portrait of Hermine Gallia", + "wikipedia_title": null + }, + { + "artsy_artwork_slug": "john-pettie-bonnie-prince-charlie-entering-the-ballroom-at-holyroodhouse", + "creators": [ + { + "artsy_artist_slug": "john-pettie", + "name": "John Pettie", + "qid": "Q3182289" + } + ], + "description": "painting by John Pettie", + "import": { + "status": "pending" + }, + "kind": "painting", + "qid": "Q28027545", + "selection_reason": "wikidata_p11005_with_english_wikipedia", + "title": "Bonnie Prince Charlie Entering the Ballroom at Holyroodhouse", + "wikipedia_title": "Bonnie Prince Charlie Entering the Ballroom at Holyroodhouse" + }, + { + "artsy_artwork_slug": "giovanni-battista-moroni-gian-gerolamo-grumelli", + "creators": [ + { + "artsy_artist_slug": "giovanni-battista-moroni", + "name": "Giovanni Battista Moroni", + "qid": "Q504949" + } + ], + "description": "painting in Palazzo Moroni, Bergamo, Italy, by Giovanni Battista Moroni", + "import": { + "status": "pending" + }, + "kind": "painting", + "qid": "Q28952741", + "selection_reason": "wikidata_p11005_with_english_wikipedia", + "title": "Portrait of Gian Gerolamo Grumelli (The Gentleman in Pink)", + "wikipedia_title": "The Knight in Pink" + }, + { + "artsy_artwork_slug": "titian-christ-and-the-adultresess", + "creators": [ + { + "artsy_artist_slug": "titian", + "name": "Titian", + "qid": "Q47551" + } + ], + "description": "painting by Titian", + "import": { + "status": "pending" + }, + "kind": "painting", + "qid": "Q3697619", + "selection_reason": "wikidata_p11005_with_english_wikipedia", + "title": "Christ and the Woman Taken in Adultery", + "wikipedia_title": "Christ and the Adulteress (Titian, Glasgow)" + }, + { + "artsy_artwork_slug": "odilon-redon-the-cyclops", + "creators": [ + { + "artsy_artist_slug": "odilon-redon", + "name": "Odilon Redon", + "qid": "Q154349" + } + ], + "description": "painting by Odilon Redon", + "import": { + "status": "pending" + }, + "kind": "painting", + "qid": "Q3793426", + "selection_reason": "wikidata_p11005_with_english_wikipedia", + "title": "The Cyclops", + "wikipedia_title": "The Cyclops (Redon)" + }, + { + "artsy_artwork_slug": "titian-isabella-deste", + "creators": [ + { + "artsy_artist_slug": "titian", + "name": "Titian", + "qid": "Q47551" + } + ], + "description": "painting by Titian", + "import": { + "status": "pending" + }, + "kind": "painting", + "qid": "Q3853901", + "selection_reason": "wikidata_p11005_with_english_wikipedia", + "title": "Portrait of Isabella d'Este", + "wikipedia_title": "Portrait of Isabella d'Este (Titian)" + }, + { + "artsy_artwork_slug": "titian-portrait-of-the-venetian-painter-giovanni-bellini", + "creators": [ + { + "artsy_artist_slug": "titian", + "name": "Titian", + "qid": "Q47551" + } + ], + "description": "painting by Titian", + "import": { + "status": "pending" + }, + "kind": "painting", + "qid": "Q3937402", + "selection_reason": "wikidata_p11005_painting", + "title": "Portrait of the Venetian Painter Giovanni Bellini (?)", + "wikipedia_title": null + }, + { + "artsy_artwork_slug": "titian-portrait-of-girolamo-fracastoro", + "creators": [ + { + "artsy_artist_slug": "titian", + "name": "Titian", + "qid": "Q47551" + } + ], + "description": "painting by Titian", + "import": { + "status": "pending" + }, + "kind": "painting", + "qid": "Q3937551", + "selection_reason": "wikidata_p11005_painting", + "title": "Portrait of Girolamo Fracastoro", + "wikipedia_title": null + }, + { + "artsy_artwork_slug": "titian-portrait-of-giacomo-dolfin", + "creators": [ + { + "artsy_artist_slug": "titian", + "name": "Titian", + "qid": "Q47551" + } + ], + "description": "painting by Titian", + "import": { + "status": "pending" + }, + "kind": "painting", + "qid": "Q3937574", + "selection_reason": "wikidata_p11005_with_english_wikipedia", + "title": "Portrait of Giacomo di Andrea Dolfin", + "wikipedia_title": "Portrait of Giacomo di Andrea Dolfin" + }, + { + "artsy_artwork_slug": "titian-diana-and-callisto", + "creators": [ + { + "artsy_artist_slug": "titian", + "name": "Titian", + "qid": "Q47551" + } + ], + "description": "painting by Titian", + "import": { + "status": "pending" + }, + "kind": "painting", + "qid": "Q4161055", + "selection_reason": "wikidata_p11005_with_english_wikipedia", + "title": "Diana and Callisto", + "wikipedia_title": "Diana and Callisto" + }, + { + "artsy_artwork_slug": "titian-cardinal-pietro-bembo", + "creators": [ + { + "artsy_artist_slug": "titian", + "name": "Titian", + "qid": "Q47551" + } + ], + "description": "painting by Titian in Washington, D.C.", + "import": { + "status": "pending" + }, + "kind": "painting", + "qid": "Q4252531", + "selection_reason": "wikidata_p11005_with_english_wikipedia", + "title": "Portrait of Cardinal Pietro Bembo", + "wikipedia_title": "Portrait of Cardinal Pietro Bembo" + }, + { + "artsy_artwork_slug": "rembrandt-van-rijn-the-anatomy-lesson-of-dr-nicolaes-tulp", + "creators": [ + { + "artsy_artist_slug": "rembrandt-van-rijn", + "name": "Rembrandt", + "qid": "Q5598" + } + ], + "description": "painting by Rembrandt von Rijn", + "import": { + "status": "pending" + }, + "kind": "painting", + "qid": "Q661378", + "selection_reason": "wikidata_p11005_with_english_wikipedia", + "title": "The Anatomy Lesson of Dr. Nicolaes Tulp", + "wikipedia_title": "The Anatomy Lesson of Dr. Nicolaes Tulp" + }, + { + "artsy_artwork_slug": "charles-demuth-i-saw-the-figure-5-in-gold", + "creators": [ + { + "artsy_artist_slug": "charles-demuth", + "name": "Charles Demuth", + "qid": "Q380494" + } + ], + "description": "painting by Charles Demuth", + "import": { + "status": "pending" + }, + "kind": "painting", + "qid": "Q6622215", + "selection_reason": "wikidata_p11005_with_english_wikipedia", + "title": "I Saw the Figure 5 in Gold", + "wikipedia_title": "I Saw the Figure 5 in Gold" + }, + { + "artsy_artwork_slug": "titian-venus-of-urbino-1", + "creators": [ + { + "artsy_artist_slug": "titian", + "name": "Titian", + "qid": "Q47551" + } + ], + "description": "painting by Titian", + "import": { + "status": "pending" + }, + "kind": "painting", + "qid": "Q727875", + "selection_reason": "wikidata_p11005_with_english_wikipedia", + "title": "Venus of Urbino", + "wikipedia_title": "Venus of Urbino" + }, + { + "artsy_artwork_slug": "titian-st-jerome-in-penitence", + "creators": [ + { + "artsy_artist_slug": "titian", + "name": "Titian", + "qid": "Q47551" + } + ], + "description": "painting by Titian in the Pinacoteca di Brera", + "import": { + "status": "pending" + }, + "kind": "painting", + "qid": "Q7401416", + "selection_reason": "wikidata_p11005_with_english_wikipedia", + "title": "Saint Jerome in Penitence", + "wikipedia_title": "Saint Jerome in Penitence (Titian, 1552)" + }, + { + "artsy_artwork_slug": "titian-christ-shown-to-the-people-ecce-homo", + "creators": [ + { + "artsy_artist_slug": "titian", + "name": "Titian", + "qid": "Q47551" + } + ], + "description": "painting by Titian in the Saint Louis Art Museum, St. Louis, Missouri", + "import": { + "status": "pending" + }, + "kind": "painting", + "qid": "Q78644404", + "selection_reason": "wikidata_p11005_painting", + "title": "Christ Shown to the People (Ecce Homo)", + "wikipedia_title": null + }, + { + "artsy_artwork_slug": "titian-venus-with-a-mirror", + "creators": [ + { + "artsy_artist_slug": "titian", + "name": "Titian", + "qid": "Q47551" + } + ], + "description": "painting by Titian in Washington, D.C.", + "import": { + "status": "pending" + }, + "kind": "painting", + "qid": "Q7920648", + "selection_reason": "wikidata_p11005_with_english_wikipedia", + "title": "Venus with a Mirror", + "wikipedia_title": "Venus with a Mirror" + }, + { + "artsy_artwork_slug": "titian-la-sultana-rossa", + "creators": [], + "description": "painting by the workshop of Titian", + "import": { + "status": "pending" + }, + "kind": "painting", + "qid": "Q79400367", + "selection_reason": "wikidata_p11005_painting", + "title": "Portrait of a Woman", + "wikipedia_title": null + }, + { + "artsy_artwork_slug": "jacopo-tintoretto-christ-at-the-sea-of-galilee", + "creators": [], + "description": "painting from the circle of Jacopo Tintoretto, possibly by Lambert Sustris", + "import": { + "status": "pending" + }, + "kind": "painting", + "qid": "Q9191534", + "selection_reason": "wikidata_p11005_with_english_wikipedia", + "title": "Christ at the Sea of Galilee", + "wikipedia_title": "Christ at the Sea of Galilee" + } + ], + "checksum": "a8f0a0c01f239e8d54d4f648e37f3d6de30271f0b3f2d7a879c527d0e3e3b398", + "generated_at": "2026-09-12T18:22:45Z", + "schema_version": 1, + "selection": { + "artwork_kinds": [ + "painting" + ], + "discovery": { + "discovered": 43, + "kinds": [ + "painting" + ], + "limit": 50, + "local": 0, + "next_offset": 57, + "remote": 43, + "requests": 2, + "truncated": false + }, + "identity": "Wikidata QID plus P11005; title is never identity", + "pilot_cap": 500, + "selection_reason": "painting with exact Artsy artwork identifier", + "source": "Wikidata P11005/P2042", + "wikipedia_required": false + }, + "updated_at": "2026-09-12T18:23:18Z" +} diff --git a/priv/artworks/meaning-mappings-v1.json b/priv/artworks/meaning-mappings-v1.json new file mode 100644 index 00000000..1489faea --- /dev/null +++ b/priv/artworks/meaning-mappings-v1.json @@ -0,0 +1,96 @@ +{ + "version": 1, + "policy": "Exact sense and language mappings only. Artsy genes are attributed source vocabulary; these rows create candidates, never accepted illustrates claims.", + "mappings": [ + { + "version": 1, + "lemma": "war", + "language": "en", + "gloss_contains": ["armed conflict", "warfare", "hostilities"], + "gene_name": "Conflict", + "match_type": "related", + "note": "A direct Conflict assignment is related evidence, not proof that the work illustrates this exact sense." + }, + { + "version": 1, + "lemma": "family", + "language": "en", + "gloss_contains": ["related", "parents", "children", "household"], + "gene_name": "Family", + "match_type": "exact", + "note": "The exact sense still requires review of the depicted relationship." + }, + { + "version": 1, + "lemma": "love", + "language": "en", + "gloss_contains": ["affection", "romantic", "attachment"], + "gene_name": "Love", + "match_type": "exact", + "note": "Provider vocabulary is candidate evidence only." + }, + { + "version": 1, + "lemma": "grief", + "language": "en", + "gloss_contains": ["sorrow", "loss", "sadness"], + "gene_name": "Grief", + "match_type": "exact", + "note": "Provider vocabulary is candidate evidence only." + }, + { + "version": 1, + "lemma": "solitude", + "language": "en", + "gloss_contains": ["alone", "seclusion", "solitary"], + "gene_name": "Solitude", + "match_type": "exact", + "note": "A title match alone never satisfies this mapping." + }, + { + "version": 1, + "lemma": "justice", + "language": "en", + "gloss_contains": ["fair", "law", "right"], + "gene_name": "Justice", + "match_type": "exact", + "note": "The allegorical or political sense must be reviewed." + }, + { + "version": 1, + "lemma": "power", + "language": "en", + "gloss_contains": ["control", "authority", "influence"], + "gene_name": "Power", + "match_type": "exact", + "note": "Wrong-sense physical-power matches must be dismissed." + }, + { + "version": 1, + "lemma": "nepotism", + "language": "en", + "gloss_contains": ["relative", "family", "favoritism", "favouritism"], + "gene_name": "Family", + "match_type": "broader", + "note": "Family is exploratory context, not evidence of favoritism or nepotism." + }, + { + "version": 1, + "lemma": "death", + "language": "en", + "gloss_contains": ["end of life", "dying", "ceasing to live"], + "gene_name": "Death", + "match_type": "exact", + "note": "A direct Death assignment proposes a candidate; the depicted event and exact lexical sense still require review." + }, + { + "version": 1, + "lemma": "freedom", + "language": "en", + "gloss_contains": ["liberty", "free", "restraint"], + "gene_name": "Freedom", + "match_type": "exact", + "note": "Political, physical, and allegorical freedom must not be conflated during review." + } + ] +} diff --git a/test/devils_dictionary/absorb/sources/wikidata_test.exs b/test/devils_dictionary/absorb/sources/wikidata_test.exs index 1042977f..e23b885b 100644 --- a/test/devils_dictionary/absorb/sources/wikidata_test.exs +++ b/test/devils_dictionary/absorb/sources/wikidata_test.exs @@ -80,6 +80,53 @@ defmodule DevilsDictionary.Absorb.Sources.WikidataTest do end describe "materialize/1" do + test "artwork and artist crosswalks keep Wikidata properties and namespaces exact" do + artwork = + %{ + "id" => "Q900086", + "labels" => %{"en" => %{"value" => "Fixture Work"}}, + "descriptions" => %{}, + "aliases" => %{}, + "sitelinks" => %{}, + "claims" => %{ + "P31" => [entity_statement("Q3305213")], + "P11005" => [string_statement("fixture-work", "normal")], + "P170" => [entity_statement("Q900087")] + } + } + + artist = + %{ + "id" => "Q900087", + "labels" => %{"en" => %{"value" => "Fixture Artist"}}, + "descriptions" => %{}, + "aliases" => %{}, + "sitelinks" => %{}, + "claims" => %{ + "P31" => [entity_statement("Q5")], + "P2042" => [string_statement("fixture-artist", "normal")] + } + } + + assert [work] = out(artwork).concepts + assert work.kind == :work + assert work.work_kind == "artwork" + assert work.metadata["wikidata_creators"] == ["Q900087"] + + assert Enum.any?( + work.external_identifiers, + &(&1.namespace == "artsy_artwork_slug" and &1.external_id == "fixture-work") + ) + + assert [person] = out(artist).concepts + assert person.kind == :person + + assert Enum.any?( + person.external_identifiers, + &(&1.namespace == "artsy_artist_slug" and &1.external_id == "fixture-artist") + ) + end + test "film records expose exact cross-provider identifiers and release year" do titanic = film_records() |> Enum.find(&(&1["id"] == "Q44578")) |> out() assert [concept] = titanic.concepts @@ -279,4 +326,14 @@ defmodule DevilsDictionary.Absorb.Sources.WikidataTest do } } end + + defp entity_statement(qid) do + %{ + "rank" => "normal", + "type" => "statement", + "mainsnak" => %{ + "datavalue" => %{"value" => %{"id" => qid}, "type" => "wikibase-entityid"} + } + } + end end diff --git a/test/devils_dictionary/artsy/client_test.exs b/test/devils_dictionary/artsy/client_test.exs new file mode 100644 index 00000000..4811c87d --- /dev/null +++ b/test/devils_dictionary/artsy/client_test.exs @@ -0,0 +1,138 @@ +defmodule DevilsDictionary.Artsy.ClientTest do + use ExUnit.Case, async: true + + alias DevilsDictionary.Artsy.Client + + test "reports stale search hits and rebuilds pagination with the artwork filter" do + responses = [ + response(201, %{"token" => "fixture-token"}), + response(200, %{ + "_embedded" => %{ + "results" => [ + %{ + "type" => "artwork", + "title" => "Missing work", + "_links" => %{ + "self" => %{"href" => "https://api.artsy.net/api/artworks/missing-work"}, + "permalink" => %{"href" => "https://www.artsy.net/artwork/missing-work"} + } + } + ] + }, + "_links" => %{ + "next" => %{"href" => "https://api.artsy.net/api/search?q=war&size=1&offset=1"} + } + }), + response(404, %{}), + response(200, %{"_embedded" => %{"results" => []}, "_links" => %{}}) + ] + + {client, calls} = client(responses) + assert {:ok, first, client} = Client.search_artworks(client, "war", size: 1) + assert [%{status: :unavailable}] = first.items + assert first.next_cursor == "1" + refute first.returned_next_preserved_filter + + assert {:ok, _second, _client} = + Client.search_artworks(client, "war", size: 1, cursor: first.next_cursor) + + requests = Agent.get(calls, &Enum.reverse/1) + second_search = Enum.at(requests, 3) + assert second_search[:params][:q] == "war" + assert second_search[:params][:type] == "artwork" + assert second_search[:params][:offset] == 1 + end + + test "caps 429 retries and counts every request" do + responses = [ + response(201, %{"token" => "fixture-token"}), + response(429, %{}) |> Req.Response.put_header("retry-after", "0"), + artwork_response("work-one") + ] + + {client, _calls} = client(responses, max_retries: 1) + assert {:ok, body, client, %{retries: 1}} = Client.artwork(client, "work-one") + assert body["slug"] == "work-one" + assert client.request_count == 3 + assert client.retry_count == 1 + end + + test "refreshes an expired token once after 401" do + responses = [ + response(201, %{"token" => "first"}), + response(401, %{}), + response(201, %{"token" => "second"}), + artwork_response("work-one") + ] + + {client, _calls} = client(responses) + assert {:ok, _body, client, _meta} = Client.artwork(client, "work-one") + assert client.token == "second" + assert client.request_count == 4 + end + + test "rejects cross-host redirects without sending credentials there" do + redirect = + response(302, %{}) |> Req.Response.put_header("location", "https://example.test/api/stolen") + + {client, calls} = client([response(201, %{"token" => "fixture-token"}), redirect]) + + assert {:error, %{code: "unsafe_redirect"}, client} = Client.artwork(client, "work-one") + assert client.request_count == 2 + assert length(Agent.get(calls, & &1)) == 2 + end + + test "fails closed when the request budget is consumed by authentication" do + {client, _calls} = client([response(201, %{"token" => "fixture-token"})], request_limit: 1) + assert {:error, %{code: "request_limit"}, client} = Client.artwork(client, "work-one") + assert client.request_count == 1 + end + + test "normalization keeps opaque ids and slugs separate" do + normalized = Client.normalize_artwork(response_body("work-one")) + assert normalized["id"] == "opaque-1" + assert normalized["slug"] == "work-one" + refute normalized["id"] == normalized["slug"] + end + + defp client(responses, opts \\ []) do + calls = start_supervised!({Agent, fn -> [] end}) + queue = start_supervised!({Agent, fn -> responses end}, id: make_ref()) + + request_fun = fn options -> + Agent.update(calls, &[options | &1]) + + Agent.get_and_update(queue, fn + [response | rest] -> {{:ok, response}, rest} + [] -> {{:error, %Req.TransportError{reason: :timeout}}, []} + end) + end + + client = + Client.new( + [ + client_id: "fixture-id", + client_secret: "fixture-secret", + request_fun: request_fun, + sleep_fun: fn _ -> :ok end, + rate_limit_ms: 0 + ] ++ opts + ) + + {client, calls} + end + + defp artwork_response(slug), do: response(200, response_body(slug)) + + defp response_body(slug) do + %{ + "id" => "opaque-1", + "slug" => slug, + "title" => "Fixture Work", + "category" => "Painting", + "_links" => %{"permalink" => %{"href" => "https://www.artsy.net/artwork/#{slug}"}} + } + end + + defp response(status, body), do: %Req.Response{status: status, body: body} +end diff --git a/test/devils_dictionary/artworks/manifest_test.exs b/test/devils_dictionary/artworks/manifest_test.exs new file mode 100644 index 00000000..463fe496 --- /dev/null +++ b/test/devils_dictionary/artworks/manifest_test.exs @@ -0,0 +1,28 @@ +defmodule DevilsDictionary.Artworks.ManifestTest do + use ExUnit.Case, async: true + + alias DevilsDictionary.Artworks.Manifest + + test "round trips a checksummed versioned manifest and rejects edits" do + path = + Path.join(System.tmp_dir!(), "artwork-manifest-#{System.unique_integer([:positive])}.json") + + candidate = %{"qid" => "Q1", "artsy_artwork_slug" => "one", "kind" => "painting"} + + manifest = [candidate, candidate] |> Manifest.new() |> Manifest.save!(path) + assert length(manifest["candidates"]) == 1 + assert Manifest.load!(path)["checksum"] == manifest["checksum"] + + edited = path |> File.read!() |> String.replace("\"one\"", "\"two\"") + File.write!(path, edited) + assert_raise ArgumentError, ~r/checksum mismatch/, fn -> Manifest.load!(path) end + end + + test "completed statuses are resumable while quota is retried" do + for status <- ~w(matched created skipped conflict unavailable) do + assert Manifest.completed?(%{"import" => %{"status" => status}}) + end + + refute Manifest.completed?(%{"import" => %{"status" => "quota_exhausted"}}) + end +end diff --git a/test/devils_dictionary/artworks/seeder_test.exs b/test/devils_dictionary/artworks/seeder_test.exs new file mode 100644 index 00000000..3151a8d0 --- /dev/null +++ b/test/devils_dictionary/artworks/seeder_test.exs @@ -0,0 +1,207 @@ +defmodule DevilsDictionary.Artworks.SeederTest do + use DevilsDictionary.DataCase, async: false + + alias DevilsDictionary.Artsy.Client + alias DevilsDictionary.Artworks + alias DevilsDictionary.Artworks.{Manifest, Seeder} + alias DevilsDictionary.Claims.Assertion + alias DevilsDictionary.Registry + alias DevilsDictionary.Registry.{Entity, ExternalIdentifier, WorkDetails} + alias DevilsDictionary.Sources.SourceRecord + alias DevilsDictionary.Sources.ReconciliationCase + + test "exact Wikidata/P11005/P2042 identities seed once and rerun without duplicate links" do + candidate = candidate() + manifest = Manifest.new([candidate]) + + {first, summary} = + Seeder.run(manifest, artsy_client: client(painting_responses()), record_limit: 1) + + assert summary.created == 1 + assert summary.creators_resolved == 1 + assert summary.creator_links == 1 + assert get_in(first, ["candidates", Access.at(0), "import", "status"]) == "created" + + work_id = Registry.by_external_id("wikidata", "Q900086") + assert work_id == Registry.by_external_id("artsy_artwork_slug", "fixture-work") + assert work_id == Registry.by_external_id("artsy_artwork_id", "opaque-work-86") + assert Repo.get!(WorkDetails, work_id).work_kind == "artwork" + + artist_id = Registry.by_external_id("wikidata", "Q900087") + assert artist_id == Registry.by_external_id("artsy_artist_slug", "fixture-artist") + assert artist_id == Registry.by_external_id("artsy_artist_id", "opaque-artist-87") + assert Repo.get!(Entity, artist_id).preferred_label == "Fixture Artist" + assert Repo.aggregate(Assertion, :count) >= 1 + + {_second, again} = + Seeder.run(Manifest.new([candidate]), + artsy_client: client(painting_responses()), + record_limit: 1 + ) + + assert again.matched == 1 + assert again.creator_links == 0 + + assert Repo.aggregate( + from(identifier in ExternalIdentifier, + where: identifier.namespace == "artsy_artwork_id" + ), + :count + ) == 1 + + assert Repo.aggregate( + from(record in SourceRecord, where: record.external_id == "artwork:opaque-work-86"), + :count + ) == 1 + end + + test "a search-linked record that is gone remains a useful Wikidata artwork" do + manifest = Manifest.new([candidate()]) + + {_manifest, summary} = + Seeder.run(manifest, artsy_client: client(unavailable_responses()), record_limit: 1) + + assert summary.unavailable == 1 + work_id = Registry.by_external_id("wikidata", "Q900086") + assert Registry.by_external_id("artsy_artwork_slug", "fixture-work") == work_id + assert Registry.by_external_id("artsy_artwork_id", "opaque-work-86") == nil + end + + test "a print/reproduction never collapses into the painting identity" do + manifest = Manifest.new([candidate()]) + + {_manifest, summary} = + Seeder.run(manifest, artsy_client: client(print_responses()), record_limit: 1) + + assert summary.skipped == 1 + assert summary.reproductions == 1 + assert Registry.by_external_id("artsy_artwork_id", "opaque-print-86") == nil + assert Registry.by_external_id("wikidata", "Q900086") + end + + test "a provider slug that contradicts Wikidata opens one review case" do + manifest = Manifest.new([candidate()]) + responses = [token(), response(200, artwork("opaque-other-86", "different-work", "Painting"))] + + {_manifest, summary} = Seeder.run(manifest, artsy_client: client(responses), record_limit: 1) + assert summary.conflicts == 1 + assert Repo.aggregate(ReconciliationCase, :count) == 1 + assert Registry.by_external_id("artsy_artwork_slug", "fixture-work") + assert Registry.by_external_id("artsy_artwork_slug", "different-work") == nil + end + + test "resume skips completed records without consuming a provider request" do + manifest = + Manifest.new([candidate()]) + |> Manifest.update_candidate(0, %{status: :unavailable, reason: "fixture"}) + + {_manifest, summary} = + Seeder.run(manifest, artsy_client: client([]), record_limit: 1, resume: true) + + assert summary.processed == 0 + assert summary.requests == 0 + end + + test "withdrawal succeeds after a seeded creator citation and keeps durable QIDs" do + {_manifest, _summary} = + Seeder.run(Manifest.new([candidate()]), + artsy_client: client(painting_responses()), + record_limit: 1 + ) + + work_id = Registry.by_external_id("wikidata", "Q900086") + artist_id = Registry.by_external_id("wikidata", "Q900087") + + assert {:ok, summary} = Artworks.withdraw_artsy("fixture provider shutdown") + assert summary.claims_withdrawn == 1 + assert summary.evidence_removed == 1 + assert summary.payloads_deleted == 2 + assert Registry.by_external_id("wikidata", "Q900086") == work_id + assert Registry.by_external_id("wikidata", "Q900087") == artist_id + assert Registry.by_external_id("artsy_artwork_id", "opaque-work-86") == nil + assert Registry.by_external_id("artsy_artist_id", "opaque-artist-87") == nil + end + + defp client(responses) do + queue = start_supervised!({Agent, fn -> responses end}, id: make_ref()) + + request_fun = fn _options -> + Agent.get_and_update(queue, fn + [response | rest] -> {{:ok, response}, rest} + [] -> {{:error, %Req.TransportError{reason: :timeout}}, []} + end) + end + + Client.new( + client_id: "fixture-id", + client_secret: "fixture-secret", + request_fun: request_fun, + sleep_fun: fn _ -> :ok end, + rate_limit_ms: 0, + request_limit: 20 + ) + end + + defp candidate do + %{ + "qid" => "Q900086", + "artsy_artwork_slug" => "fixture-work", + "title" => "Fixture Work", + "description" => "A selected fixture painting", + "wikipedia_title" => "Fixture Work", + "kind" => "painting", + "selection_reason" => "test_exact_p11005", + "creators" => [ + %{ + "qid" => "Q900087", + "name" => "Fixture Artist", + "artsy_artist_slug" => "fixture-artist" + } + ] + } + end + + defp painting_responses do + [ + token(), + response(200, artwork("opaque-work-86", "fixture-work", "Painting")), + response(200, %{ + "_embedded" => %{ + "artists" => [ + %{ + "id" => "opaque-artist-87", + "slug" => "fixture-artist", + "name" => "Fixture Artist", + "_links" => %{ + "permalink" => %{"href" => "https://www.artsy.net/artist/fixture-artist"} + } + } + ] + } + }), + response(200, %{ + "_embedded" => %{ + "genes" => [%{"id" => "gene-war", "name" => "Conflict", "type" => "subject"}] + } + }) + ] + end + + defp unavailable_responses, do: [token(), response(404, %{})] + + defp print_responses, + do: [token(), response(200, artwork("opaque-print-86", "fixture-work", "Print"))] + + defp artwork(id, slug, category) do + %{ + "id" => id, + "slug" => slug, + "title" => "Fixture Work", + "category" => category, + "_links" => %{"permalink" => %{"href" => "https://www.artsy.net/artwork/#{slug}"}} + } + end + + defp token, do: response(201, %{"token" => "fixture-token"}) + defp response(status, body), do: %Req.Response{status: status, body: body} +end diff --git a/test/devils_dictionary_web/live/artwork_live_test.exs b/test/devils_dictionary_web/live/artwork_live_test.exs new file mode 100644 index 00000000..5e93fd48 --- /dev/null +++ b/test/devils_dictionary_web/live/artwork_live_test.exs @@ -0,0 +1,132 @@ +defmodule DevilsDictionaryWeb.ArtworkLiveTest do + use DevilsDictionaryWeb.ConnCase, async: false + + import DevilsDictionary.WordFixtures + + alias DevilsDictionary.{Artworks, Claims, Fixtures, Registry, Repo, Sources} + alias DevilsDictionary.Claims.AssertionEvidence + alias DevilsDictionary.Corpus.SourceRecordRevision + alias DevilsDictionary.Sources.{MaterializedOutput, Source} + + setup ctx do + %{sources: sources, scopes: scopes} = Fixtures.seed_catalog!() + + {:ok, work} = + Registry.create_work(%{ + preferred_label: "The Fixture of War", + description: "A painting used to verify the reusable artwork flow.", + work_kind: "artwork" + }) + + {:ok, creator} = Registry.create_person(%{preferred_label: "Fixture Painter"}) + {:ok, claim} = Claims.assert(work.object_id, "authored_by", creator.object_id) + {:ok, _} = Registry.add_external_id(work.object_id, "wikidata", "Q860086") + {:ok, _} = Registry.add_external_id(work.object_id, "artsy_artwork_slug", "fixture-of-war") + + {:ok, record} = + Sources.upsert_record(sources["artsy"], %{ + external_id: "artwork:opaque-live-86", + url: "https://www.artsy.net/artwork/fixture-of-war", + raw: %{ + "artwork" => %{ + "id" => "opaque-live-86", + "slug" => "fixture-of-war", + "title" => "The Fixture of War", + "permalink" => "https://www.artsy.net/artwork/fixture-of-war" + }, + "genes" => [%{"id" => "gene-conflict", "name" => "Conflict"}] + } + }) + + Repo.insert!(%MaterializedOutput{ + source_record_id: record.id, + output_role: "concept", + output_key: "Q860086", + output_object_id: work.object_id + }) + + revision = Claims.current_revision(claim.id) + + {:ok, _evidence} = + Claims.add_evidence(revision.id, %{ + source_record_revision_id: record.current_revision.id, + evidence_role: :supports, + attribution_text: "Fixture provider citation" + }) + + Map.merge(ctx, %{ + sources: sources, + animals: scopes["animals"], + work: work, + creator: creator, + record: record, + claim: claim + }) + end + + test "catalog search links both the work and its creator", ctx do + {:ok, view, _html} = live(ctx.conn, ~p"/artworks") + assert has_element?(view, "#artwork-#{ctx.work.object_id}") + + assert has_element?( + view, + "#artwork-#{ctx.work.object_id} a[href^='/entities/#{ctx.work.object_id}/']" + ) + + assert has_element?( + view, + "#artwork-#{ctx.work.object_id} a[href^='/entities/#{ctx.creator.object_id}/']" + ) + + html = view |> form("#artwork-search", search: %{q: "Fixture Painter"}) |> render_change() + assert html =~ ~s(id="artwork-#{ctx.work.object_id}") + + html = view |> form("#artwork-search", search: %{q: "no such work"}) |> render_change() + refute html =~ ~s(id="artwork-#{ctx.work.object_id}") + end + + test "a contributor opens the existing composer with the exact artwork selected", ctx do + %{conn: conn, user: user} = register_and_log_in_user(%{conn: ctx.conn}) + _user = Repo.update!(Ecto.Changeset.change(user, internal_contributor: true)) + + {:ok, view, _html} = live(conn, ~p"/connect?subject=#{ctx.work.object_id}") + assert has_element?(view, "#composer-subject-chosen") + assert render(view) =~ "The Fixture of War" + assert has_element?(view, "#predicate-illustrates") + end + + test "a direct gene produces a labeled candidate for the exact sense, never a claim", ctx do + war = word!(ctx, "war", ["wordnet"]) + sense = sense!(ctx, war, "wordnet", gloss: "organized armed conflict or warfare") + + {:ok, view, _html} = live(ctx.conn, ~p"/define/war") + assert has_element?(view, "#artwork-candidates") + + assert has_element?( + view, + "#artwork-candidate-#{ctx.work.object_id}-#{sense.object_id}" + ) + + assert render(view) =~ "not yet reviewed" + end + + test "provider withdrawal deletes Artsy payloads but preserves Wikidata identity", ctx do + assert {:ok, summary} = Artworks.withdraw_artsy("fixture terms ended") + assert summary.payloads_deleted == 1 + assert summary.records_disabled == 1 + assert summary.evidence_removed == 1 + + assert Registry.by_external_id("wikidata", "Q860086") == ctx.work.object_id + assert Repo.get!(Source, ctx.sources["artsy"].id).active == false + assert Repo.get(SourceRecordRevision, ctx.record.current_revision.id) == nil + assert Repo.aggregate(AssertionEvidence, :count) == 0 + assert Claims.current_revision(ctx.claim.id).lifecycle_state == :active + + record = Repo.get!(DevilsDictionary.Sources.SourceRecord, ctx.record.id) + refute record.display_allowed + assert record.url == nil + assert record.content_hash == nil + assert Artworks.search("Fixture") |> Enum.any?(&(&1.object_id == ctx.work.object_id)) + assert Artworks.get(ctx.work.object_id).artsy == nil + end +end diff --git a/test/devils_dictionary_web/live/home_live_test.exs b/test/devils_dictionary_web/live/home_live_test.exs index 46156445..1e92d834 100644 --- a/test/devils_dictionary_web/live/home_live_test.exs +++ b/test/devils_dictionary_web/live/home_live_test.exs @@ -56,7 +56,7 @@ defmodule DevilsDictionaryWeb.HomeLiveTest do # template and the sentence is what is under test, not its indentation. stats = live |> element("#stats") |> render() |> squash() - assert stats =~ "3 words indexed · 2 with at least one definition · 8 sources so far" + assert stats =~ "3 words indexed · 2 with at least one definition · 9 sources so far" refute stats =~ "animals enriched" refute stats =~ "Animals" From 19daf6bbdb26f49b2eafe7e4aee663a4ffb25f66 Mon Sep 17 00:00:00 2001 From: Holden Thomas Date: Sat, 12 Sep 2026 21:40:37 +0200 Subject: [PATCH 2/8] Repair issue 86 Artsy audit findings --- config/config.exs | 1 + config/test.exs | 5 + docs/audits/2026-09-12-issue86-repair.md | 110 ++ .../artsy-feasibility-2026-09-12.md | 62 +- docs/integrations/artsy.md | 63 +- .../absorb/sources/wikidata.ex | 10 +- lib/devils_dictionary/application.ex | 1 + lib/devils_dictionary/artsy/availability.ex | 104 ++ lib/devils_dictionary/artsy/client.ex | 357 ++++-- .../artsy/request_coordinator.ex | 94 ++ lib/devils_dictionary/artworks.ex | 293 ++++- lib/devils_dictionary/artworks/manifest.ex | 57 +- lib/devils_dictionary/artworks/seeder.ex | 1092 +++++++++++------ .../artworks/wikidata_candidates.ex | 156 ++- .../components/artwork.ex | 35 +- .../live/artwork_live.ex | 118 +- .../live/connection_live.ex | 68 +- lib/devils_dictionary_web/live/entity_live.ex | 77 +- lib/mix/tasks/dd.artworks.seed.ex | 58 +- .../manifests/repair-new-entry-v3.json | 65 + priv/artworks/manifests/repair-pilot-v2.json | 140 +++ priv/artworks/meaning-mappings-v1.json | 144 ++- .../absorb/sources/wikidata_test.exs | 8 +- test/devils_dictionary/artsy/client_test.exs | 44 +- .../artworks/seeder_test.exs | 225 +++- .../live/artwork_live_test.exs | 115 +- 26 files changed, 2870 insertions(+), 632 deletions(-) create mode 100644 docs/audits/2026-09-12-issue86-repair.md create mode 100644 lib/devils_dictionary/artsy/availability.ex create mode 100644 lib/devils_dictionary/artsy/request_coordinator.ex create mode 100644 priv/artworks/manifests/repair-new-entry-v3.json create mode 100644 priv/artworks/manifests/repair-pilot-v2.json diff --git a/config/config.exs b/config/config.exs index acafba3e..ce5f9aea 100644 --- a/config/config.exs +++ b/config/config.exs @@ -135,6 +135,7 @@ config :devils_dictionary, :giphy, config :devils_dictionary, :artsy, endpoint: "https://api.artsy.net", rate_limit_ms: 340, + freshness_seconds: 24 * 60 * 60, enabled: true # Oban (#69 §5). `absorb: 1` because a dump absorb is a single long stream. diff --git a/config/test.exs b/config/test.exs index ffe1734f..340630ce 100644 --- a/config/test.exs +++ b/config/test.exs @@ -37,6 +37,11 @@ config :devils_dictionary, :giphy, rating: "g", enabled: false +# Test clients use deterministic injected transports. Focused coordinator tests +# start their own named process; transaction rollbacks must not leave the global +# withdrawal generation disabled for unrelated cases. +config :devils_dictionary, :artsy, coordinator: nil + config :devils_dictionary, :discovery_req_options, plug: {Req.Test, DevilsDictionary.Discovery.Providers.CineGraph} diff --git a/docs/audits/2026-09-12-issue86-repair.md b/docs/audits/2026-09-12-issue86-repair.md new file mode 100644 index 00000000..bf689c0b --- /dev/null +++ b/docs/audits/2026-09-12-issue86-repair.md @@ -0,0 +1,110 @@ +# Issue #86 independent-audit repair + +Date: 2026-09-12 + +Branch: `codex/issue-86-artsy-catalog` + +Pull request: [#98](https://github.com/razrfly/dictionary/pull/98) + +This is the completion ledger for the findings in the independent C+ audit. It +distinguishes verified behavior from unsupported coverage and should be used for the +next independent review. + +## Audit finding disposition + +| Finding | Repair | Evidence | +| --- | --- | --- | +| Preserve current Wikidata evidence | The seeder loads/materializes an existing full source record and never replaces it with manifest-shaped data. Missing records are either hydrated by the shared adapter or represented as explicit manifest evidence without synthesizing a Wikidata observation. | Regression retains byte-for-byte P18/claims via `Sources.raw/1`. The old external audit assertion reads a `load_in_query: false` field directly and must use that accessor. | +| Honor creator review decisions | Cards and creator search use `Claims.visible(:public)`, require `authored_by`, and deduplicate creator identities. | Rejected creator and unrelated-predicate search regressions. | +| Stop access on withdrawal | One configuration/source/coordinator gate is checked at every client attempt and at importer/UI entry points. Provider writes lock the same source row as withdrawal; generation invalidation discards an in-flight response. | Post-withdrawal import reports zero calls, zero processed records, and no new source records. | +| Keep partial imports resumable | Manifest schema v2 checkpoints Wikidata, artwork, artists, genes, identity, and creators. Artist/gene pages persist their cursor after every page. Temporary/request/quota failures remain partial. | A two-attempt run checkpoints after artwork, resume finishes collections, and a second resume uses zero calls without duplicate IDs. | +| Preserve independent creators | P170 is materialized as `authored_by`; local and remote candidate pages merge creator QIDs/slugs; creator identities/links are created before optional Artsy hydration. | Missing-Artsy and multi-page/multi-creator regressions; live Q122978100 page links Ferdinand Hodler after Artsy 404. | +| Shared limits/freshness | A supervised application-wide coordinator paces all actual attempts, shares full Retry-After deadlines, counts auth/retry/redirect attempts, and invalidates withdrawal generations. Source caches carry 24-hour freshness metadata and completed stale records are refreshable. | Negative monotonic epoch, concurrent-client pacing, long Retry-After, retry, 401 and bounded-budget tests. | +| Exact mappings | Registry v2 pins exact WordNet source keys and opaque Artsy gene IDs with version, relation type, note, and probe evidence. | Four mappings enabled; six failed gene-resource probes remain disabled. Same-name/wrong-ID assignment produces no suggestion. | +| Complete reader/catalog flow | Local title/creator/retained-tag search is network-free and paginated; cards/pages use independent images first, Artsy references only while active, show rights/freshness/source data, and link creators. Candidate links preselect exact endpoints, predicate, revision, locator and rationale in the existing composer. | Pagination, tag search, rejected visibility, rich-page, creator-link, and composer regressions plus desktop/mobile browser checks. | + +## Live bounded evidence + +The complete feasibility campaign is bounded at **no more than 189 Artsy attempts**, +below the issue's 200-request ceiling. No image bytes were downloaded and no secret or +token was printed, saved, or committed. + +| Run | Wikidata requests | Artsy attempts | Result | +| --- | ---: | ---: | --- | +| Readiness + original implementation evidence | documented separately | 160 | 20-query readiness, five-record pilot/rerun, browser lookup | +| Stable gene-ID repair probes | 0 | 16 | Conflict, Family and Love resolved; six candidate slugs returned 404 | +| Interrupted legacy-cache repair | 0 | at most 3 | Exposed/fixed resume and monotonic-coordinator paths; no claimed import outcome | +| Two-record repair pilot | 1 | 7 | matched 2; existing Artsy-rich entry reused 1; existing Wikidata entry newly Artsy-hydrated 1; creator links 1; conflicts/reproductions/quota 0 | +| Same two-record manifest resume | skipped | 0 | processed 0; duplicates 0 | +| New-entry proof Q122978100 | 1 | 3 | Wikidata created the absent artwork + creator; exact Artsy endpoint unavailable; no provider data invented | +| Same new-entry manifest resume | skipped | 0 | processed 0; duplicates 0 | + +The two-record pilot's Artsy summary was `matched: 2, created: 0, unavailable: 0`. +The new-entry proof's Artsy summary was `matched: 0, created: 0, unavailable: 1`. +Creation of *Les Âmes Déçues* and Ferdinand Hodler happened in the explicitly reported +Wikidata hydration stage. This avoids misreporting “a newly useful local entry” as an +Artsy-created record. + +## Browser acceptance + +Development Chrome was checked at desktop and a narrow mobile width against real +retained pilot data: + +- `/artworks` showed seven reusable works after the final proof, independently licensed + thumbnails where present, honest image fallbacks elsewhere, creator/source links, + and responsive layout. +- Local search reduced the catalog to the one requested Ranuccio record without a + provider call. +- *Portrait of Ranuccio Farnese* showed the permitted image, creator, date, medium, + collection, image credit, Wikidata/Wikipedia/Artsy links, and cache notice. +- *Les Âmes Déçues*, created during the final proof, showed the retained Wikidata + description, Ferdinand Hodler link and Wikidata source; the unavailable Artsy image + remained an honest fallback. Creator navigation returned to the work. +- An authenticated internal contributor opened the composer with exact Cupid and + WordNet `war` identities, `illustrates`, the immutable Artsy revision, opaque Conflict + gene locator, and explanatory rationale preselected. No claim was submitted. + +## Product and retention decision + +Artsy remains **optional, freshness-limited discovery/enrichment**, not the durable +collection authority. Wikidata provides exact cross-source identity; museum/open +sources provide durable metadata/images where their record permits it. The 20-query +Met comparison hydrated 48/48 requested records and found 30 public-domain image +references, versus 3/59 hydrated Artsy search hits and no blanket permanent Artsy +retention basis. Provider withdrawal removes provider payloads, genes, opaque IDs, +URLs and source-owned claims while preserving independently supported identities, +creator relations, open fields, and editorial work. + +## Acceptance status + +- [x] Full P0 capability/rights/coverage report and explicit GO-for-optional-enrichment decision. +- [x] Versioned selected corpus plus bounded Wikidata P11005/P2042 discovery and import task. +- [x] Dry-run, limits, atomic manifest checkpoints, resume and duplicate-free reruns. +- [x] Exact identifiers, conflicts, reproductions, missing endpoints, retries, 401, 429/local quota, pagination and provider withdrawal covered. +- [x] Rich reusable catalog, local pagination/search, artwork/creator navigation and honest image fallback. +- [x] Exact meaning candidates remain distinct from artwork identity and require normal evidence/review. +- [x] Desktop/mobile local search, artwork pages, creator links and select/review composer verified. +- [x] Open museum comparison, field-level retention matrix and provider-disappearance operation documented. +- [ ] Reviewed/featured art examples gained: **0**. Four exact mappings can produce candidates where a retained direct assignment exists, but no human review was performed or implied by this issue implementation. +- [ ] Permanent Artsy retention rights: **not established**. The supported fallback is deliberately implemented; Artsy-only material remains removable. + +The last two unchecked statements are reported product facts, not silently omitted +engineering work. They must not be converted to claims of reviewed coverage or durable +Artsy ownership without separate evidence/authorization. + +## Verification commands + +- Focused repository regressions: **48 tests, 0 failures** across the Artsy client, + seeder, Wikidata adapter and artwork LiveViews. +- Original external audit file: **4 of 5 expectations pass**. Its remaining line reads + `SourceRecordRevision.payload` directly even though issue #93 intentionally declares + that 660 MB aggregate field `load_in_query: false`. The equivalent repository + regression uses `Sources.raw/1`, verifies exact payload equality (including P18), and + passes. Re-enabling payloads on every ordinary revision query would regress the shared + identity foundation's memory-safety fix. +- Full suite through the required `mix precommit`: **934 tests, 0 failures**. The two + ignored corpus archives were temporarily linked read-only from the original + Dictionary checkout so the real source-manifest digests were verified; the links + were removed immediately afterward and are not branch artifacts. +- `mix format --check-formatted`, `mix compile --warnings-as-errors`, and + `git diff --check`: pass. diff --git a/docs/integrations/artsy-feasibility-2026-09-12.md b/docs/integrations/artsy-feasibility-2026-09-12.md index 8d13a630..a80d1ac4 100644 --- a/docs/integrations/artsy-feasibility-2026-09-12.md +++ b/docs/integrations/artsy-feasibility-2026-09-12.md @@ -24,6 +24,11 @@ title search. Widen only after the pilot reports its actual coverage. - Completion probe: 93 Artsy requests, including 2 token exchanges; no retries and no 429 responses. Combined issue evidence: **132 Artsy requests**, below the 200-request initial-probe ceiling. +- The implementation-validation runs added at most 57 requests: 28 for the original + five-record import/rerun/browser check, 16 bounded gene-resource probes, at most 3 + in an interrupted legacy-cache repair run, 7 in the two-record repair pilot, and 3 + in the one-record new-entry proof. The cumulative ceiling is therefore **189 Artsy + requests**, still below 200. Both final manifest resumes made zero provider requests. - Open-source comparison: 68 Met requests against the current paginated `/public/collection/v1.1/search` endpoint and object hydration endpoint. - No image bytes were requested or downloaded. @@ -49,7 +54,62 @@ title search. Widen only after the pilot reports its actual coverage. | P2042 artist crosswalk | Wikidata P2042 is an artist slug | `vincent-van-gogh` redirected, then returned opaque ID `4d8b92944eb68a1b2c000264` | Resolve creators through exact slug/ID evidence, never through a name string. | | Direct gene assignments | Artwork links expose assigned genes | Goya returned 5 assignments: 19th Century, Chiaroscuro, Collective History, Conflict, Cultural Commentary | Preserve as attributed Artsy vocabulary. No string-to-meaning equivalence. | | Gene-filtered acquisition | Gene resources link to artworks | The first 10 `gene_id=19th Century` artwork IDs were identical to the unfiltered first 10 | **Unverified/unsafe.** Do not seed through gene traversal. A direct per-work assignment may label a review candidate only. | -| Retry/quota signal | 5 requests/sec documented | Probe throttled below 3/sec; 0 retries, 0 quota responses | One shared bounded client handles 429/5xx with capped exponential delay and reports exhaustion. | +| Retry/quota signal | 5 requests/sec documented | Probe throttled below 3/sec; 0 retries, 0 quota responses | One application-wide coordinator paces all callers. It honors the full `Retry-After`, bounds retries and total attempts, and reports local request exhaustion separately from provider quota exhaustion. | + +## Repair feasibility findings + +The independent audit correctly identified that human-readable gene names were not +stable identifiers. A final bounded resource probe resolved four mappings to exact +opaque Artsy gene IDs: + +| Mapping | Exact Artsy gene ID | Exact Dictionary meaning | Status | +| --- | --- | --- | --- | +| Conflict | `4db9b645db68d133c600123e` | `oewn-00975181-n#war` | enabled, related | +| Family | `50b99005c2f9a4a28b00015a` | `oewn-07986853-n#family` | enabled, related | +| Family | `50b99005c2f9a4a28b00015a` | `oewn-01157387-n#nepotism` | enabled, broader association with an explicit review note | +| Love | `4de292fcef72520001005fe5` | `oewn-07558676-n#love` | enabled, related | + +The same probe returned 404 for the candidate slugs `grief`, `solitude`, `justice`, +`power`, `death`, and `freedom`. Those six definitions remain in the versioned +registry as disabled evidence records; the product does not pretend that their names +are IDs. Consequently the supported mapping set is four exact mappings, not ten. + +The final repair pilot used one bounded Wikidata request and seven Artsy attempts for +two manifest records. Both exact identities matched. One previously enriched record +was reused and one existing Wikidata artwork was newly hydrated with Artsy metadata; +no local identity was created, no conflict/reproduction/quota event occurred, and one +creator link was added. A resume of the saved manifest processed zero records, made +zero Artsy requests, and created no duplicates. + +A final one-candidate proof started from Q122978100, which was absent from the local +catalog. One bounded Wikidata request materialized *Les Âmes Déçues* and its creator, +Ferdinand Hodler, as reusable local identities with an `authored_by` route and source +link. The exact Artsy endpoint was unavailable after three attempts, so no Artsy +fields or image were invented. Its saved manifest records `unavailable`; resuming it +made zero requests and no duplicates. This deliberately separates “new local identity +created from independent Wikidata evidence” from the Artsy import summary's honest +`created: 0, unavailable: 1`. + +The pilot also exposed and fixed a coordinator initialization bug: monotonic clocks +may start at a negative value, so an absolute zero deadline could sleep indefinitely. +The coordinator now initializes from the injected/current monotonic time, with a +negative-epoch regression test. + +### Live-audit findings and enforced behavior + +1. Search hits whose artwork endpoint returns 404 are retained only as an explicit + unavailable discovery result. They never become an identity or claim. +2. Search and collection pagination never follows a provider URL verbatim. The client + validates scheme/host/path and reconstructs the next request with the immutable + `type=artwork` or `artwork_id` filter. +3. Gene traversal remains disabled. Only direct assignments returned by an exact, + hydrated artwork can meet a versioned opaque-ID mapping. +4. Every import validates category and exact Wikidata P11005/P2042 crosswalks. + Prints/reproductions and conflicting slugs are skipped or opened for reconciliation; + titles never merge records. +5. API access is treated as freshness-limited cache permission, not permanent rights. + Provider payloads, URLs, genes, opaque IDs, and provider-only claims can be removed + while independently supported Wikidata/Wikipedia identity and open imagery remain. The three hydrated search records were: diff --git a/docs/integrations/artsy.md b/docs/integrations/artsy.md index ffcb4124..b17bc94c 100644 --- a/docs/integrations/artsy.md +++ b/docs/integrations/artsy.md @@ -15,8 +15,10 @@ never identity keys. The public `/artworks` page searches the local catalog without network calls. Its separate Artsy form is an explicit, transient source-availability check. -Word pages use only retained direct gene assignments and a versioned set of ten -exact sense/language mappings to display **not-yet-reviewed candidates**. A contributor +Word pages use only retained direct gene assignments and a versioned mapping registry +to display **not-yet-reviewed candidates**. Four mappings currently have verified +opaque Artsy gene IDs; six researched names remain explicitly disabled because their +bounded resource probes returned 404. A contributor selects a local work and exact meaning in the existing `/connect` composer; normal attribution, evidence, review, revision, and dispute rules apply. @@ -36,8 +38,11 @@ ARTSY_CLIENT_SECRET If either is absent, the provider is disabled while the local collection keeps working. The client exchanges them for an in-memory XAPP token, refreshes once -after a 401, follows at most two same-host HTTPS `/api/` redirects, throttles -requests, caps retries, obeys `Retry-After`, and returns scrubbed error codes. +after a 401, follows at most two same-host HTTPS `/api/` redirects, and returns +scrubbed error codes. One application-wide coordinator serializes and paces every +auth, redirect, retry, artwork, artist, gene, and search attempt. It honors the full +`Retry-After`, invalidates in-flight generations on withdrawal, and distinguishes a +local request ceiling from a provider quota response. ## Bulk seed runbook @@ -62,14 +67,24 @@ Run or resume a bounded import: ```sh mix dd.artworks.seed \ --manifest priv/artworks/manifests/pilot-v1.json \ - --record-limit 50 --request-limit 160 --batch-size 10 --resume + --record-limit 50 --request-limit 160 --batch-size 10 \ + --wikidata-entity-limit 100 --wikidata-request-limit 4 --resume ``` +Before Artsy enrichment, the task hydrates selected artwork and creator QIDs through +the existing bounded Wikidata adapter so useful encyclopedia pages do not depend on +Artsy availability. `--skip-wikidata-hydration` is for a verified resume whose +Wikidata stage is already complete; `--refresh-wikidata` asks the shared adapter to +refresh instead of using current source records. `--dry-run` performs neither +database writes nor provider calls. + The JSON contains a schema version, selection policy, source identifiers, -selection reasons, creator crosswalks, checksum, and per-record outcome. It -contains no credentials. It is saved atomically after each record, so an -interrupted run can resume. Completed records are skipped with `--resume`; -quota-stopped records remain retryable. Re-running without `--resume` is also +selection reasons, creator crosswalks, checksum, and per-record outcome plus +independent Wikidata, artwork, artist, gene, identity, and creator-link stage +checkpoints. It contains no credentials. It is saved atomically after every +page/stage transition, so an interrupted collection page can resume from its cursor. +Completed records are skipped with `--resume`; request- or quota-stopped records +remain retryable. Re-running without `--resume` is also safe: exact IDs converge through `SourceIdentity`, source records upsert, and creator claims use a source-native origin key. @@ -96,11 +111,36 @@ errors. A final browser search used two more requests and returned an honest empty state. Readiness, both import runs, and browser validation used 160 Artsy requests, below the issue's 200-request feasibility ceiling. +The independent-audit repair added 16 bounded gene-resource attempts, at most 3 +attempts in one interrupted legacy-cache run, 7 attempts in the two-record pilot, +and 3 attempts in a final new-entry proof. The cumulative ceiling is therefore +**189 Artsy attempts**. The two-record repair pilot +made one Wikidata request, matched two exact artwork identities, reused one already +hydrated artwork, newly hydrated one existing Wikidata artwork, added one creator +link, and created zero local identities. Its saved manifest then resumed with zero +processed records, zero provider requests, and zero duplicates. + +The final one-record proof began with Q122978100 absent from the catalog. One bounded +Wikidata request created the reusable *Les Âmes Déçues* artwork and Ferdinand Hodler +creator identities and connected them. Its exact Artsy endpoint was unavailable after +three attempts, so the page truthfully uses Wikidata description/creator/source data +and an image fallback. The Artsy summary is separately and accurately `created: 0, +unavailable: 1`; resuming that manifest also made zero calls and no duplicates. + +Browser verification used real permitted retained data at desktop and in a narrow +mobile-width Chrome window. It covered the six-work local catalog, local search to +one result, the rich *Portrait of Ranuccio Farnese* page, navigation to Titian, and +the authenticated review composer preselected with the exact Cupid artwork, exact +WordNet `war` sense, `illustrates` predicate, immutable Artsy source-record revision, +opaque gene locator, and rationale. No review claim was submitted. + The detailed audit and museum comparison are in [`artsy-feasibility-2026-09-12.md`](artsy-feasibility-2026-09-12.md). It records the 10-work/10-meaning probe, unsafe gene-filter finding, pagination defect, 404 rate, exact P11005/P2042 validation, and 20-query Metropolitan Museum -comparison. +comparison. The independent-audit finding ledger, final live manifests, and browser +acceptance are in +[`2026-09-12-issue86-repair.md`](../audits/2026-09-12-issue86-repair.md). ## Retention and shutdown @@ -127,7 +167,8 @@ editorial claims remain. The operation is transactional and tested. expected source unavailability, not a reason to weaken identity matching. - Artsy's `gene_id` search filter did not change the sampled result page, so gene traversal/filter acquisition is disabled. Only direct genes from an - exact hydrated artwork are considered. + exact hydrated artwork are considered. Four exact mappings are enabled; the six + unresolved resource slugs remain documented and disabled. - The Metropolitan Museum API remains the better open-image/reference source, but its broad text search is noisy and four audit concepts had no results. - The committed manifest is a reproducible seed set, not an assertion that all diff --git a/lib/devils_dictionary/absorb/sources/wikidata.ex b/lib/devils_dictionary/absorb/sources/wikidata.ex index bea4fd11..4aff5ba2 100644 --- a/lib/devils_dictionary/absorb/sources/wikidata.ex +++ b/lib/devils_dictionary/absorb/sources/wikidata.ex @@ -56,7 +56,13 @@ defmodule DevilsDictionary.Absorb.Sources.Wikidata do # narrowed), and L3 only ever asks about `parent_taxon`. @parent_properties ~w(P171 P13176) @general_relation_properties ~w(P31 P279) - @relation_types %{"P171" => :parent_taxon, "P279" => :subclass_of, "P31" => :instance_of} + @relation_types %{ + "P171" => :parent_taxon, + "P279" => :subclass_of, + "P31" => :instance_of, + "P170" => :authored_by + } + @artwork_relation_properties ~w(P170) @closure_relation_types Map.put(@relation_types, "P13176", :taxon_item) # Taxonomic chains run long: species → genus → … → Animalia passes through @@ -497,7 +503,7 @@ defmodule DevilsDictionary.Absorb.Sources.Wikidata do properties = if explicit_qids(opts) == [], do: @parent_properties, - else: @parent_properties ++ @general_relation_properties + else: @parent_properties ++ @general_relation_properties ++ @artwork_relation_properties Enum.flat_map(properties, &Client.entity_ids(entity, &1)) end diff --git a/lib/devils_dictionary/application.ex b/lib/devils_dictionary/application.ex index 6286c853..958bddaf 100644 --- a/lib/devils_dictionary/application.ex +++ b/lib/devils_dictionary/application.ex @@ -12,6 +12,7 @@ defmodule DevilsDictionary.Application do DevilsDictionary.Repo, {DNSCluster, query: Application.get_env(:devils_dictionary, :dns_cluster_query) || :ignore}, {Phoenix.PubSub, name: DevilsDictionary.PubSub}, + DevilsDictionary.Artsy.RequestCoordinator, {Oban, Application.fetch_env!(:devils_dictionary, Oban)}, # Health figures that are seconds of queries (the scorecard) are cached # here so a page can show them without recomputing them on every mount. diff --git a/lib/devils_dictionary/artsy/availability.ex b/lib/devils_dictionary/artsy/availability.ex new file mode 100644 index 00000000..aafd29dc --- /dev/null +++ b/lib/devils_dictionary/artsy/availability.ex @@ -0,0 +1,104 @@ +defmodule DevilsDictionary.Artsy.Availability do + @moduledoc """ + One operational gate for every Artsy entry point. + + Configuration can disable the integration before the database is available; + once the source catalog exists, a withdrawn source also disables it. Callers + that write provider data must use `with_active_source/1`, which locks the + source row so withdrawal and publication cannot cross in flight. + """ + + import Ecto.Query + + alias DevilsDictionary.Repo + alias DevilsDictionary.Sources.Source + + @slug "artsy" + + @doc "Whether credentials, configuration, source lifecycle and coordinator permit access." + def enabled? do + configured?() and source_active?() and coordinator_enabled?() + rescue + DBConnection.OwnershipError -> configured?() and coordinator_enabled?() + end + + @doc "Whether the server configuration contains both credentials without exposing them." + def configured? do + config = config() + + config[:enabled] != false and present?(config[:client_id]) and + present?(config[:client_secret]) + end + + @doc "Returns a stable, scrubbed reason when the provider cannot be used." + def status do + config = config() + + status_with_credentials( + config[:client_id], + config[:client_secret], + Keyword.get(config, :coordinator, DevilsDictionary.Artsy.RequestCoordinator) + ) + end + + @doc "The lifecycle check for a client carrying explicit server-side credentials." + def status_with_credentials(client_id, client_secret) do + status_with_credentials(client_id, client_secret, DevilsDictionary.Artsy.RequestCoordinator) + end + + def status_with_credentials(client_id, client_secret, coordinator) do + cond do + config()[:enabled] == false -> {:error, :configured_off} + not (present?(client_id) and present?(client_secret)) -> {:error, :credentials_missing} + not source_active?() -> {:error, :source_withdrawn} + not coordinator_enabled?(coordinator) -> {:error, :source_withdrawn} + true -> :ok + end + rescue + DBConnection.OwnershipError -> + if(present?(client_id) and present?(client_secret), + do: :ok, + else: {:error, :credentials_missing} + ) + end + + @doc "Runs a provider-data write while holding the same source lock as withdrawal." + def with_active_source(fun) when is_function(fun, 1) do + Repo.transaction(fn -> + source = + Repo.one(from source in Source, where: source.slug == @slug, lock: "FOR UPDATE") + + if source && source.active && config()[:enabled] != false && coordinator_enabled?() do + fun.(source) + else + Repo.rollback(:provider_disabled) + end + end) + end + + defp source_active? do + case Repo.get_by(Source, slug: @slug) do + nil -> true + source -> source.active + end + end + + defp coordinator_enabled? do + coordinator = + Keyword.get(config(), :coordinator, DevilsDictionary.Artsy.RequestCoordinator) + + coordinator_enabled?(coordinator) + end + + defp coordinator_enabled?(nil), do: true + + defp coordinator_enabled?(coordinator) do + case Process.whereis(coordinator) do + nil -> true + _pid -> DevilsDictionary.Artsy.RequestCoordinator.enabled?(coordinator) + end + end + + defp config, do: Application.get_env(:devils_dictionary, :artsy, []) + defp present?(value), do: is_binary(value) and String.trim(value) != "" +end diff --git a/lib/devils_dictionary/artsy/client.ex b/lib/devils_dictionary/artsy/client.ex index 851b275b..0c064638 100644 --- a/lib/devils_dictionary/artsy/client.ex +++ b/lib/devils_dictionary/artsy/client.ex @@ -29,18 +29,29 @@ defmodule DevilsDictionary.Artsy.Client do rate_limit_ms: @default_rate_limit_ms, request_fun: nil, sleep_fun: nil, - now_fun: nil + now_fun: nil, + availability_fun: nil, + coordinator: DevilsDictionary.Artsy.RequestCoordinator @type t :: %__MODULE__{} @doc "Creates a client. Missing credentials produce an explicit authentication failure on use." def new(opts \\ []) do config = Application.get_env(:devils_dictionary, :artsy, []) + client_id = opts[:client_id] || config[:client_id] + client_secret = opts[:client_secret] || config[:client_secret] + + coordinator = + Keyword.get( + opts, + :coordinator, + Keyword.get(config, :coordinator, DevilsDictionary.Artsy.RequestCoordinator) + ) %__MODULE__{ base_url: opts[:base_url] || config[:endpoint] || @base_url, - client_id: opts[:client_id] || config[:client_id], - client_secret: opts[:client_secret] || config[:client_secret], + client_id: client_id, + client_secret: client_secret, request_limit: positive!(opts[:request_limit] || @default_request_limit, :request_limit), max_retries: non_negative!(opts[:max_retries] || @default_retries, :max_retries), timeout_ms: positive!(opts[:timeout_ms] || @default_timeout, :timeout_ms), @@ -51,7 +62,17 @@ defmodule DevilsDictionary.Artsy.Client do ), request_fun: opts[:request_fun] || (&Req.request/1), sleep_fun: opts[:sleep_fun] || (&Process.sleep/1), - now_fun: opts[:now_fun] || (&DateTime.utc_now/0) + now_fun: opts[:now_fun] || (&DateTime.utc_now/0), + availability_fun: + opts[:availability_fun] || + fn -> + DevilsDictionary.Artsy.Availability.status_with_credentials( + client_id, + client_secret, + coordinator + ) + end, + coordinator: coordinator } end @@ -59,6 +80,10 @@ defmodule DevilsDictionary.Artsy.Client do def configured?(%__MODULE__{} = client), do: present?(client.client_id) and present?(client.client_secret) + @doc "True only while configuration and the persisted source lifecycle permit requests." + def available?(%__MODULE__{} = client), + do: configured?(client) and client.availability_fun.() == :ok + @doc "Hydrates one artwork by opaque API ID or validated slug." def artwork(client, identifier) do with :ok <- valid_identifier(identifier) do @@ -80,13 +105,36 @@ defmodule DevilsDictionary.Artsy.Client do @doc "Returns every creator on one hydrated artwork page, bounded by `size`." def artwork_artists(client, artwork_id, opts \\ []) do size = page_size(opts[:size] || @max_page_size) - authorized(client, :get, "/api/artists", params: [artwork_id: artwork_id, size: size]) + offset = cursor_offset(opts[:cursor]) + + with :ok <- valid_identifier(artwork_id), + true <- offset != :error do + collection_page(client, :artists, artwork_id, size, offset) + else + _ -> {:error, failure(:invalid_cursor, nil, "/api/artists"), client} + end end @doc "Returns direct gene assignments for one work. This is not gene traversal." def artwork_genes(client, artwork_id, opts \\ []) do size = page_size(opts[:size] || @max_page_size) - authorized(client, :get, "/api/genes", params: [artwork_id: artwork_id, size: size]) + offset = cursor_offset(opts[:cursor]) + + with :ok <- valid_identifier(artwork_id), + true <- offset != :error do + collection_page(client, :genes, artwork_id, size, offset) + else + _ -> {:error, failure(:invalid_cursor, nil, "/api/genes"), client} + end + end + + @doc "Fetches one gene by opaque ID or slug for bounded feasibility verification." + def gene(client, identifier) do + with :ok <- valid_identifier(identifier) do + authorized(client, :get, "/api/genes/#{identifier}", []) + else + {:error, reason} -> {:error, failure(reason, nil, "/api/genes"), client} + end end @doc """ @@ -122,7 +170,7 @@ defmodule DevilsDictionary.Artsy.Client do query: query, items: items, mixed_type_count: Enum.count(hits, &(&1["type"] != "artwork")), - next_cursor: next_cursor(next_url, query, size), + next_cursor: next_cursor(client, next_url, query, size), returned_next_preserved_filter: query_param(next_url, "type") == "artwork", request_meta: meta }, client} @@ -144,6 +192,7 @@ defmodule DevilsDictionary.Artsy.Client do "date" => body["date"], "collecting_institution" => body["collecting_institution"], "image_rights" => body["image_rights"], + "description" => body["blurb"] || body["description"], "thumbnail_url" => link(body, "thumbnail"), "permalink" => public_link(body, "permalink"), "artists_url" => api_link(body, "artists"), @@ -230,9 +279,11 @@ defmodule DevilsDictionary.Artsy.Client do end defp authorized_url(client, method, url, opts) do - with {:ok, client} <- ensure_token(client) do + with :ok <- provider_available(client), + {:ok, client} <- ensure_token(client) do do_authorized(client, method, url, opts, false) else + {:error, failure} -> {:error, failure, client} {:error, failure, client} -> {:error, failure, client} end end @@ -294,82 +345,174 @@ defmodule DevilsDictionary.Artsy.Client do end defp request(client, method, url, opts, retry_number, redirects) do - if client.request_count >= client.request_limit do - {:error, failure(:request_limit, nil, URI.parse(url).path), client} - else - client.sleep_fun.(client.rate_limit_ms) - client = %{client | request_count: client.request_count + 1} - - request_opts = - [ - method: method, - url: url, - redirect: false, - retry: false, - receive_timeout: client.timeout_ms, - connect_options: [timeout: client.timeout_ms] - ] ++ opts - - case client.request_fun.(request_opts) do - {:ok, %Req.Response{status: status} = response} - when status in [429, 500, 502, 503, 504] and retry_number < client.max_retries -> - delay = retry_delay(response, retry_number) - client.sleep_fun.(delay) + cond do + client.request_count >= client.request_limit -> + {:error, failure(:request_limit, nil, URI.parse(url).path), client} - request( - %{client | retry_count: client.retry_count + 1}, - method, - url, - opts, - retry_number + 1, - redirects - ) + provider_available(client) != :ok -> + {:error, failure(:provider_disabled, nil, URI.parse(url).path), client} - {:ok, %Req.Response{status: status} = response} - when status in 300..399 and redirects < 2 -> - case redirect_url(response) do - redirect when is_binary(redirect) -> - if safe_api_url?(client, redirect) do - request( - client, - :get, - redirect, - Keyword.delete(opts, :json), - retry_number, - redirects + 1 - ) - else - {:error, failure(:unsafe_redirect, status, URI.parse(url).path), client} - end - - _ -> - {:ok, response, client, request_meta(retry_number, redirects)} + true -> + with {:ok, generation, wait_ms} <- acquire(client) do + client.sleep_fun.(wait_ms) + client = %{client | request_count: client.request_count + 1} + + request_opts = + [ + method: method, + url: url, + redirect: false, + retry: false, + receive_timeout: client.timeout_ms, + connect_options: [timeout: client.timeout_ms] + ] ++ opts + + response = client.request_fun.(request_opts) + + if generation_current?(client, generation) and provider_available(client) == :ok do + handle_response( + response, + client, + method, + url, + opts, + retry_number, + redirects + ) + else + {:error, failure(:provider_disabled, nil, URI.parse(url).path), client} end + else + {:error, :provider_disabled} -> + {:error, failure(:provider_disabled, nil, URI.parse(url).path), client} + end + end + end - {:ok, response} -> - {:ok, response, client, request_meta(retry_number, redirects)} + defp handle_response( + {:ok, %Req.Response{status: 429} = response}, + client, + method, + url, + opts, + retry_number, + redirects + ) do + case DevilsDictionary.Discovery.Transport.retry_after_seconds(response) do + seconds when is_integer(seconds) and seconds > 0 -> + defer(client, seconds * 1_000) + + {:ok, response, client, + request_meta(retry_number, redirects) |> Map.put(:retry_after_seconds, seconds)} - {:error, _reason} when retry_number < client.max_retries -> - client.sleep_fun.(retry_delay(nil, retry_number)) + _ -> + retry_response(response, client, method, url, opts, retry_number, redirects) + end + end + defp handle_response( + {:ok, %Req.Response{status: status} = response}, + client, + method, + url, + opts, + retry_number, + redirects + ) + when status in [500, 502, 503, 504] do + retry_response(response, client, method, url, opts, retry_number, redirects) + end + + defp handle_response( + {:ok, %Req.Response{status: status} = response}, + client, + _method, + url, + opts, + retry_number, + redirects + ) + when status in 300..399 and redirects < 2 do + case redirect_url(response) do + redirect when is_binary(redirect) -> + if safe_api_url?(client, redirect) do request( - %{client | retry_count: client.retry_count + 1}, - method, - url, - opts, - retry_number + 1, - redirects + client, + :get, + redirect, + Keyword.delete(opts, :json), + retry_number, + redirects + 1 ) + else + {:error, failure(:unsafe_redirect, status, URI.parse(url).path), client} + end - {:error, reason} -> - {:error, - failure( - transport_code(reason), - nil, - URI.parse(url).path, - request_meta(retry_number, redirects) - ), client} - end + _ -> + {:ok, response, client, request_meta(retry_number, redirects)} + end + end + + defp handle_response({:ok, response}, client, _method, _url, _opts, retry_number, redirects), + do: {:ok, response, client, request_meta(retry_number, redirects)} + + defp handle_response({:error, reason}, client, method, url, opts, retry_number, redirects) do + if retry_number < client.max_retries do + client.sleep_fun.(retry_delay(nil, retry_number)) + + request( + %{client | retry_count: client.retry_count + 1}, + method, + url, + opts, + retry_number + 1, + redirects + ) + else + {:error, + failure( + transport_code(reason), + nil, + URI.parse(url).path, + request_meta(retry_number, redirects) + ), client} + end + end + + defp retry_response(response, client, method, url, opts, retry_number, redirects) do + if retry_number < client.max_retries do + client.sleep_fun.(retry_delay(response, retry_number)) + + request( + %{client | retry_count: client.retry_count + 1}, + method, + url, + opts, + retry_number + 1, + redirects + ) + else + {:ok, response, client, request_meta(retry_number, redirects)} + end + end + + defp collection_page(client, kind, artwork_id, size, offset) do + path = if kind == :artists, do: "/api/artists", else: "/api/genes" + + case authorized(client, :get, path, + params: [artwork_id: artwork_id, size: size, offset: offset] + ) do + {:ok, body, client, meta} -> + next_url = get_in(body, ["_links", "next", "href"]) + + {:ok, body, client, + Map.merge(meta, %{ + next_cursor: collection_cursor(client, next_url, path, artwork_id), + returned_next_preserved_filter: query_param(next_url, "artwork_id") == artwork_id + })} + + error -> + error end end @@ -379,11 +522,13 @@ defmodule DevilsDictionary.Artsy.Client do present?(token) and DateTime.compare(expires, DateTime.add(now_fun.(), 60, :second)) == :gt end - defp next_cursor(url, query, _size) do + defp next_cursor(client, url, query, _size) do with true <- is_binary(url), uri <- URI.parse(url), + expected <- URI.parse(client.base_url), true <- - uri.scheme == "https" and uri.host == "api.artsy.net" and uri.path == "/api/search", + uri.scheme == "https" and uri.host == expected.host and uri.port == expected.port and + uri.path == "/api/search", params <- URI.decode_query(uri.query || ""), true <- params["q"] == query, {offset, ""} when offset >= 0 <- Integer.parse(params["offset"] || "") do @@ -447,14 +592,56 @@ defmodule DevilsDictionary.Artsy.Client do defp query_param(url, key), do: URI.parse(url).query |> then(&URI.decode_query(&1 || "")) |> Map.get(key) - defp retry_delay(response, retry_number) do - retry_after = - if response, do: DevilsDictionary.Discovery.Transport.retry_after_seconds(response) + defp retry_delay(_response, retry_number), + do: min(trunc(:math.pow(2, retry_number)) * 1_000, 10_000) + + defp request_meta(retries, redirects), do: %{retries: retries, redirects: redirects} + + defp collection_cursor(client, url, path, artwork_id) do + with true <- is_binary(url), + uri <- URI.parse(url), + expected <- URI.parse(client.base_url), + true <- uri.scheme == "https" and uri.host == expected.host and uri.path == path, + params <- URI.decode_query(uri.query || ""), + true <- params["artwork_id"] in [nil, artwork_id], + {offset, ""} when offset >= 0 <- Integer.parse(params["offset"] || "") do + Integer.to_string(offset) + else + _ -> nil + end + end + + defp provider_available(client) do + case client.availability_fun.() do + :ok -> :ok + {:error, reason} -> {:error, failure(reason, nil, "/api")} + _ -> {:error, failure(:provider_disabled, nil, "/api")} + end + end + + defp acquire(%{coordinator: nil}), do: {:ok, 0, 0} - min((retry_after || trunc(:math.pow(2, retry_number))) * 1_000, 10_000) + defp acquire(client) do + DevilsDictionary.Artsy.RequestCoordinator.acquire(client.coordinator, client.rate_limit_ms) + catch + :exit, _ -> {:ok, 0, client.rate_limit_ms} end - defp request_meta(retries, redirects), do: %{retries: retries, redirects: redirects} + defp generation_current?(%{coordinator: nil}, _generation), do: true + + defp generation_current?(client, generation) do + DevilsDictionary.Artsy.RequestCoordinator.current?(generation, client.coordinator) + catch + :exit, _ -> true + end + + defp defer(%{coordinator: nil}, _milliseconds), do: :ok + + defp defer(client, milliseconds) do + DevilsDictionary.Artsy.RequestCoordinator.defer(milliseconds, client.coordinator) + catch + :exit, _ -> :ok + end defp parse_datetime(value) when is_binary(value) do case DateTime.from_iso8601(value) do @@ -476,13 +663,17 @@ defmodule DevilsDictionary.Artsy.Client do defp transport_code(_), do: :provider_unavailable defp failure(code, status, path, meta \\ %{}) do - %{ + failure = %{ code: to_string(code), status: status, path: path, retries: meta[:retries] || 0, redirects: meta[:redirects] || 0 } + + if meta[:retry_after_seconds], + do: Map.put(failure, :retry_after_seconds, meta[:retry_after_seconds]), + else: failure end defp user_agent, do: Application.fetch_env!(:devils_dictionary, :user_agent) diff --git a/lib/devils_dictionary/artsy/request_coordinator.ex b/lib/devils_dictionary/artsy/request_coordinator.ex new file mode 100644 index 00000000..d0676f47 --- /dev/null +++ b/lib/devils_dictionary/artsy/request_coordinator.ex @@ -0,0 +1,94 @@ +defmodule DevilsDictionary.Artsy.RequestCoordinator do + @moduledoc """ + Application-wide pacing and withdrawal generation for Artsy HTTP attempts. + + Every authentication, redirect and retry reserves a slot here before it is + sent. The returned generation is checked again after the response, so a + request already on the wire when withdrawal begins cannot publish its body. + """ + + use GenServer + + @default_name __MODULE__ + + def start_link(opts \\ []) do + name = Keyword.get(opts, :name, @default_name) + + if name, + do: GenServer.start_link(__MODULE__, opts, name: name), + else: GenServer.start_link(__MODULE__, opts) + end + + def acquire(server \\ @default_name, interval_ms \\ nil) do + GenServer.call(server, {:acquire, interval_ms}) + end + + def current?(generation, server \\ @default_name) do + GenServer.call(server, {:current?, generation}) + end + + def defer(milliseconds, server \\ @default_name) + when is_integer(milliseconds) and milliseconds >= 0 do + GenServer.call(server, {:defer, milliseconds}) + end + + def disable(server \\ @default_name), do: GenServer.call(server, :disable) + def enable(server \\ @default_name), do: GenServer.call(server, :enable) + def enabled?(server \\ @default_name), do: GenServer.call(server, :enabled?) + def stats(server \\ @default_name), do: GenServer.call(server, :stats) + + @impl true + def init(opts) do + config = Application.get_env(:devils_dictionary, :artsy, []) + now_fun = Keyword.get(opts, :now_fun, fn -> System.monotonic_time(:millisecond) end) + + {:ok, + %{ + enabled: Keyword.get(opts, :enabled, config[:enabled] != false), + generation: 0, + next_at: now_fun.(), + attempts: 0, + interval_ms: Keyword.get(opts, :interval_ms, config[:rate_limit_ms] || 340), + now_fun: now_fun + }} + end + + @impl true + def handle_call({:acquire, _interval_ms}, _from, %{enabled: false} = state), + do: {:reply, {:error, :provider_disabled}, state} + + def handle_call({:acquire, interval_ms}, _from, state) do + now = state.now_fun.() + wait_ms = max(state.next_at - now, 0) + interval_ms = if is_integer(interval_ms), do: max(interval_ms, 0), else: state.interval_ms + + state = %{ + state + | next_at: now + wait_ms + interval_ms, + attempts: state.attempts + 1 + } + + {:reply, {:ok, state.generation, wait_ms}, state} + end + + def handle_call({:current?, generation}, _from, state), + do: {:reply, state.enabled and state.generation == generation, state} + + def handle_call({:defer, milliseconds}, _from, state) do + not_before = state.now_fun.() + milliseconds + {:reply, :ok, %{state | next_at: max(state.next_at, not_before)}} + end + + def handle_call(:disable, _from, state) do + state = %{state | enabled: false, generation: state.generation + 1} + {:reply, {:ok, state.generation}, state} + end + + def handle_call(:enable, _from, state), + do: {:reply, :ok, %{state | enabled: true, next_at: 0}} + + def handle_call(:enabled?, _from, state), do: {:reply, state.enabled, state} + + def handle_call(:stats, _from, state), + do: {:reply, Map.take(state, [:enabled, :generation, :attempts, :next_at]), state} +end diff --git a/lib/devils_dictionary/artworks.ex b/lib/devils_dictionary/artworks.ex index 517cbeaa..8eb8a22e 100644 --- a/lib/devils_dictionary/artworks.ex +++ b/lib/devils_dictionary/artworks.ex @@ -6,6 +6,7 @@ defmodule DevilsDictionary.Artworks do alias DevilsDictionary.Claims alias DevilsDictionary.Claims.{Assertion, AssertionEvidence, AssertionRevision} alias DevilsDictionary.Corpus.SourceRecordRevision + alias DevilsDictionary.Discovery alias DevilsDictionary.Discovery.{Mapping, Result, Run} alias DevilsDictionary.Encyclopedia @@ -21,15 +22,29 @@ defmodule DevilsDictionary.Artworks do alias DevilsDictionary.Repo alias DevilsDictionary.Sources.{Actor, MaterializedOutput, Source, SourceRecord} - @catalog_limit 50 + @catalog_limit 24 + @catalog_max 100 @candidate_scan_limit 500 @mapping_path Application.app_dir(:devils_dictionary, "priv/artworks/meaning-mappings-v1.json") @doc "Searches reusable local artwork identities by title or creator name. No provider call." def search(query \\ "", opts \\ []) do + catalog(query, opts).items + end + + @doc "Returns a bounded local catalog page with stable pagination metadata." + def catalog(query \\ "", opts \\ []) do query = String.trim(query || "") - limit = opts[:limit] || @catalog_limit + limit = (opts[:limit] || @catalog_limit) |> max(1) |> min(@catalog_max) + page = (opts[:page] || 1) |> max(1) + offset = (page - 1) * limit pattern = "%#{escape_like(query)}%" + gene_matches = artsy_gene_matches(pattern) + + visible_creators = + AssertionRevision + |> where([revision], revision.is_current and revision.lifecycle_state == :active) + |> Claims.visible(:public) base = from entity in Entity, @@ -37,22 +52,43 @@ defmodule DevilsDictionary.Artworks do on: object.id == entity.object_id and object.lifecycle_state == :active, join: details in WorkDetails, on: details.entity_id == entity.object_id and details.work_kind == "artwork", - left_join: revision in AssertionRevision, + left_join: revision in subquery(visible_creators), on: revision.subject_object_id == entity.object_id and revision.is_current, left_join: predicate in Claims.Predicate, on: predicate.id == revision.predicate_id and predicate.key == "authored_by", left_join: creator in Entity, - on: creator.object_id == revision.object_object_id, + on: + creator.object_id == revision.object_object_id and + not is_nil(predicate.id), + left_join: gene_match in subquery(gene_matches), + on: gene_match.object_id == entity.object_id, where: ^(query == "") or ilike(entity.preferred_label, ^pattern) or - ilike(creator.preferred_label, ^pattern), + ilike(creator.preferred_label, ^pattern) or not is_nil(gene_match.object_id), distinct: entity.object_id, - order_by: [asc: entity.preferred_label, asc: entity.object_id], - limit: ^limit, - select: entity + select: entity.object_id + + count = Repo.aggregate(base, :count) + + entities = + Repo.all( + from entity in Entity, + join: candidate in subquery(base), + on: candidate.object_id == entity.object_id, + order_by: [asc: entity.preferred_label, asc: entity.object_id], + limit: ^limit, + offset: ^offset, + select: entity + ) - entities = Repo.all(base) - views(entities) + %{ + items: views(entities), + count: count, + page: page, + page_size: limit, + previous_page: if(page > 1, do: page - 1), + next_page: if(offset + length(entities) < count, do: page + 1) + } end def get(object_id) when is_integer(object_id) do @@ -72,8 +108,6 @@ defmodule DevilsDictionary.Artworks do @doc "Returns local, exact-sense suggestions from direct retained Artsy gene assignments." def suggestions(lexeme_ids) when is_list(lexeme_ids) do - mappings = meaning_mappings() - senses = Repo.all( from sense in Sense, @@ -90,14 +124,27 @@ defmodule DevilsDictionary.Artworks do } ) + sense_ids = Enum.map(senses, & &1.id) + + mappings = + Repo.all( + from mapping in Mapping, + join: source in Source, + on: source.id == mapping.source_id and source.slug == "artsy" and source.active, + where: + mapping.enabled and mapping.operation == "artwork_gene_candidate" and + mapping.target_object_id in ^sense_ids, + select: mapping + ) + assignments = active_artsy_assignments() catalog = Map.new(search("", limit: @candidate_scan_limit), &{&1.object_id, &1}) for sense <- senses, mapping <- mappings, - mapping_matches?(mapping, sense), + mapping.target_object_id == sense.id, assignment <- assignments, - gene_matches?(mapping, assignment.genes), + gene_matches?(mapping.parameters["gene_id"], assignment.genes), artwork = catalog[assignment.object_id], not is_nil(artwork) do %{ @@ -105,14 +152,16 @@ defmodule DevilsDictionary.Artworks do sense_id: sense.id, meaning: sense.gloss, language: sense.language, - match_type: mapping["match_type"], + mapping_id: mapping.id, + source_record_revision_id: assignment.source_record_revision_id, + match_type: mapping.parameters["match_type"], match_reason: %{ provider: "Artsy", kind: "direct_gene_assignment", - gene_id: mapping["gene_id"], - gene_name: mapping["gene_name"], - mapping_version: mapping["version"], - note: mapping["note"] + gene_id: mapping.parameters["gene_id"], + gene_name: mapping.parameters["gene_name"], + mapping_version: mapping.version, + note: mapping.parameters["note"] }, review_state: :not_yet_reviewed } @@ -120,6 +169,69 @@ defmodule DevilsDictionary.Artworks do |> Enum.uniq_by(&{&1.artwork.object_id, &1.sense_id, &1.match_reason.gene_id}) end + @doc "Installs versioned Artsy-gene recipes for exact, stable source senses." + def install_meaning_mappings! do + %{sources: sources} = DevilsDictionary.Sources.Catalog.seed!() + source = Map.fetch!(sources, "artsy") + actor = mapping_actor!() + + meaning_mappings() + |> Enum.reduce(%{installed: 0, unchanged: 0, unavailable: 0}, fn definition, stats -> + if definition["enabled"] == true and is_binary(definition["gene_id"]) do + case exact_sense(definition["sense_source_slug"], definition["sense_external_key"]) do + nil -> + %{stats | unavailable: stats.unavailable + 1} + + sense_id -> + key = "artsy-gene:#{definition["gene_id"]}:sense:#{sense_id}" + + parameters = %{ + "gene_id" => definition["gene_id"], + "gene_name" => definition["gene_name"], + "match_type" => definition["match_type"], + "note" => definition["note"], + "mapping_review_status" => definition["mapping_review_status"], + "mapping_review_actor" => definition["mapping_review_actor"], + "mapping_reviewed_at" => definition["mapping_reviewed_at"], + "mapping_review_basis" => definition["mapping_review_basis"], + "sense_source_slug" => definition["sense_source_slug"], + "sense_external_key" => definition["sense_external_key"], + "candidate_only" => true + } + + case Repo.one( + from mapping in Mapping, + where: mapping.mapping_key == ^key and mapping.enabled, + order_by: [desc: mapping.version], + limit: 1 + ) do + %Mapping{ + target_object_id: ^sense_id, + operation: "artwork_gene_candidate", + parameters: ^parameters + } -> + %{stats | unchanged: stats.unchanged + 1} + + _ -> + {:ok, _mapping} = + Discovery.create_mapping_version(key, %{ + target_object_id: sense_id, + source_id: source.id, + operation: "artwork_gene_candidate", + parameters: parameters, + configured_by_actor_id: actor.id, + enabled: true + }) + + %{stats | installed: stats.installed + 1} + end + end + else + %{stats | unavailable: stats.unavailable + 1} + end + end) + end + @doc """ Disables Artsy and removes its disallowed payloads and projections. @@ -129,6 +241,8 @@ defmodule DevilsDictionary.Artworks do physically removed. """ def withdraw_artsy(reason) when is_binary(reason) and byte_size(reason) > 0 do + _ = DevilsDictionary.Artsy.RequestCoordinator.disable() + Repo.transaction(fn -> source = Repo.one!(from source in Source, where: source.slug == "artsy", lock: "FOR UPDATE") actor = withdrawal_actor!() @@ -151,7 +265,9 @@ defmodule DevilsDictionary.Artworks do from identifier in ExternalIdentifier, where: identifier.source_record_revision_id in ^revision_ids and - identifier.namespace in ["artsy_artwork_id", "artsy_artist_id"], + (identifier.namespace in ["artsy_artwork_id", "artsy_artist_id"] or + (identifier.namespace in ["artsy_artwork_slug", "artsy_artist_slug"] and + fragment("?->>'asserted_by' = 'artsy'", identifier.metadata))), lock: "FOR UPDATE" ) @@ -276,38 +392,92 @@ defmodule DevilsDictionary.Artworks do %{ object_id: entity.object_id, title: entity.preferred_label, - description: entity.description, - image_url: view.image_url, + description: entity.description || (source && source.artwork["description"]), + image_url: view.image_url || (source && source.artwork["thumbnail_url"]), + image_attribution: view.image_attribution || (source && source.artwork["image_rights"]), qid: view.qid, wikipedia_title: view.wikipedia_title, creators: Map.get(creators, entity.object_id, []), artsy: source && source.artwork, genes: (source && source.genes) || [], - source_record_id: source && source.source_record_id + source_record_id: source && source.source_record_id, + source_record_revision_id: source && source.source_record_revision_id, + date: source && source.artwork["date"], + medium: source && source.artwork["medium"], + collection: source && source.artwork["collecting_institution"], + freshness: source && source.freshness, + source_links: + Enum.reject( + [ + view.qid && %{label: "Wikidata", url: "https://www.wikidata.org/wiki/#{view.qid}"}, + view.wikipedia_title && + %{ + label: "Wikipedia", + url: "https://en.wikipedia.org/wiki/#{URI.encode(view.wikipedia_title)}" + }, + source && source.artwork["permalink"] && + %{label: "Artsy", url: source.artwork["permalink"]} + ], + &is_nil/1 + ) } end) end + defp artsy_gene_matches(pattern) do + from output in MaterializedOutput, + join: record in SourceRecord, + on: record.id == output.source_record_id and record.display_allowed, + join: source in Source, + on: source.id == record.source_id and source.slug == "artsy" and source.active, + join: revision in SourceRecordRevision, + on: revision.source_record_id == record.id and revision.revision_key == record.content_hash, + where: is_nil(output.retired_at), + where: + fragment( + "EXISTS (SELECT 1 FROM jsonb_array_elements(COALESCE(?->'genes', '[]'::jsonb)) AS gene WHERE gene->>'name' ILIKE ?)", + revision.payload, + ^pattern + ), + distinct: output.output_object_id, + select: %{object_id: output.output_object_id} + end + defp creators([]), do: %{} defp creators(work_ids) do - Repo.all( - from revision in AssertionRevision, - join: predicate in Claims.Predicate, - on: predicate.id == revision.predicate_id and predicate.key == "authored_by", - join: creator in Entity, - on: creator.object_id == revision.object_object_id, - join: object in Object, - on: object.id == creator.object_id and object.lifecycle_state == :active, - where: - revision.subject_object_id in ^work_ids and revision.is_current and - revision.lifecycle_state == :active, - order_by: [asc: creator.preferred_label, asc: creator.object_id], - select: - {revision.subject_object_id, - %{object_id: creator.object_id, label: creator.preferred_label}} - ) + query = + AssertionRevision + |> where( + [revision], + revision.subject_object_id in ^work_ids and revision.is_current and + revision.lifecycle_state == :active + ) + |> Claims.visible(:public) + |> join(:inner, [revision], predicate in Claims.Predicate, + as: :artwork_predicate, + on: predicate.id == revision.predicate_id and predicate.key == "authored_by" + ) + |> join(:inner, [revision], creator in Entity, + as: :artwork_creator, + on: creator.object_id == revision.object_object_id + ) + |> join(:inner, [artwork_creator: creator], object in Object, + on: object.id == creator.object_id and object.lifecycle_state == :active + ) + |> order_by([artwork_creator: creator], + asc: creator.preferred_label, + asc: creator.object_id + ) + |> select( + [revision, artwork_creator: creator], + {revision.subject_object_id, + %{object_id: creator.object_id, label: creator.preferred_label}} + ) + + Repo.all(query) |> Enum.group_by(&elem(&1, 0), &elem(&1, 1)) + |> Map.new(fn {work_id, creators} -> {work_id, Enum.uniq_by(creators, & &1.object_id)} end) end defp artsy_details([]), do: %{} @@ -329,8 +499,10 @@ defmodule DevilsDictionary.Artworks do select: %{ object_id: output.output_object_id, source_record_id: record.id, + source_record_revision_id: revision.id, artwork: revision.payload["artwork"], - genes: revision.payload["genes"] + genes: revision.payload["genes"], + freshness: revision.payload["freshness"] } ) |> Enum.uniq_by(& &1.object_id) @@ -349,30 +521,43 @@ defmodule DevilsDictionary.Artworks do select: details.entity_id ) ) - |> Enum.map(fn {object_id, detail} -> %{object_id: object_id, genes: detail.genes || []} end) + |> Enum.map(fn {object_id, detail} -> + %{ + object_id: object_id, + genes: detail.genes || [], + source_record_revision_id: detail.source_record_revision_id + } + end) end defp meaning_mappings do @mapping_path |> File.read!() |> Jason.decode!() |> Map.fetch!("mappings") end - defp mapping_matches?(mapping, sense) do - mapping["lemma"] == sense.lemma and mapping["language"] == sense.language and - gloss_matches?(mapping["gloss_contains"] || [], sense.gloss) - end + defp gene_matches?(gene_id, genes) when is_binary(gene_id), + do: Enum.any?(genes, &(&1["id"] == gene_id)) - defp gloss_matches?([], _gloss), do: true + defp gene_matches?(_gene_id, _genes), do: false - defp gloss_matches?(terms, gloss) do - down = String.downcase(gloss || "") - Enum.any?(terms, &String.contains?(down, String.downcase(&1))) + defp exact_sense(source_slug, external_key) do + Repo.one( + from sense in Sense, + join: source in Source, + on: source.id == sense.source_id, + where: source.slug == ^source_slug and sense.external_key == ^external_key, + select: sense.object_id + ) end - defp gene_matches?(mapping, genes) do - Enum.any?(genes, fn gene -> - (is_binary(mapping["gene_id"]) and gene["id"] == mapping["gene_id"]) or - (is_binary(mapping["gene_name"]) and gene["name"] == mapping["gene_name"]) - end) + defp mapping_actor! do + attrs = %{ + actor_kind: :import, + label: "Artsy meaning mapping registry", + metadata: %{"operation" => "artwork_gene_candidate"} + } + + Repo.get_by(Actor, actor_kind: :import, label: attrs.label) || + Repo.insert!(%Actor{} |> Actor.changeset(attrs)) end defp withdrawal_actor! do diff --git a/lib/devils_dictionary/artworks/manifest.ex b/lib/devils_dictionary/artworks/manifest.ex index 749c5b02..67e8e009 100644 --- a/lib/devils_dictionary/artworks/manifest.ex +++ b/lib/devils_dictionary/artworks/manifest.ex @@ -1,7 +1,8 @@ defmodule DevilsDictionary.Artworks.Manifest do @moduledoc "Versioned, credential-free artwork seed manifests with resumable per-record outcomes." - @schema_version 1 + @schema_version 2 + @terminal_statuses ~w(matched created skipped conflict unavailable) def new(candidates, metadata \\ %{}) when is_list(candidates) do %{ @@ -11,7 +12,12 @@ defmodule DevilsDictionary.Artworks.Manifest do "candidates" => candidates |> Enum.uniq_by(&{&1["qid"], &1["artsy_artwork_slug"]}) - |> Enum.map(&Map.put_new(&1, "import", %{"status" => "pending"})) + |> Enum.map( + &Map.put_new(&1, "import", %{ + "status" => "pending", + "stages" => default_stages() + }) + ) } |> put_checksum() end @@ -19,7 +25,8 @@ defmodule DevilsDictionary.Artworks.Manifest do def load!(path) do manifest = path |> File.read!() |> Jason.decode!() - unless manifest["schema_version"] == @schema_version and is_list(manifest["candidates"]) do + unless manifest["schema_version"] in [1, @schema_version] and + is_list(manifest["candidates"]) do raise ArgumentError, "unsupported artwork seed manifest" end @@ -28,7 +35,7 @@ defmodule DevilsDictionary.Artworks.Manifest do unless manifest["checksum"] == expected, do: raise(ArgumentError, "artwork manifest checksum mismatch") - manifest + upgrade(manifest) end def save!(manifest, path) do @@ -44,15 +51,16 @@ defmodule DevilsDictionary.Artworks.Manifest do def update_candidate(manifest, index, outcome) when is_integer(index) and is_map(outcome) do candidates = List.update_at(manifest["candidates"], index, fn candidate -> - Map.put(candidate, "import", stringify(outcome)) + previous = candidate["import"] || %{} + Map.put(candidate, "import", deep_merge(previous, stringify(outcome))) end) manifest |> Map.put("candidates", candidates) |> put_checksum() end - def completed?(candidate), - do: - get_in(candidate, ["import", "status"]) in ~w(matched created skipped conflict unavailable) + def completed?(candidate), do: get_in(candidate, ["import", "status"]) in @terminal_statuses + + def terminal_statuses, do: @terminal_statuses def checksum(value) do :sha256 @@ -72,6 +80,39 @@ defmodule DevilsDictionary.Artworks.Manifest do } end + defp default_stages do + %{ + "wikidata" => %{"status" => "pending"}, + "artwork" => %{"status" => "pending"}, + "artists" => %{"status" => "pending", "next_cursor" => nil}, + "genes" => %{"status" => "pending", "next_cursor" => nil}, + "identity" => %{"status" => "pending"}, + "creators" => %{"status" => "pending"} + } + end + + defp upgrade(%{"schema_version" => @schema_version} = manifest), do: manifest + + defp upgrade(manifest) do + manifest + |> Map.put("schema_version", @schema_version) + |> Map.update!("candidates", fn candidates -> + Enum.map(candidates, fn candidate -> + import = candidate["import"] || %{"status" => "pending"} + Map.put(candidate, "import", Map.put_new(import, "stages", default_stages())) + end) + end) + |> put_checksum() + end + + defp deep_merge(left, right) when is_map(left) and is_map(right) do + Map.merge(left, right, fn _key, left_value, right_value -> + if is_map(left_value) and is_map(right_value), + do: deep_merge(left_value, right_value), + else: right_value + end) + end + defp stringify(map) do Map.new(map, fn {key, value} -> {to_string(key), stringify_value(value)} end) end diff --git a/lib/devils_dictionary/artworks/seeder.ex b/lib/devils_dictionary/artworks/seeder.ex index 110d9f98..6c28508e 100644 --- a/lib/devils_dictionary/artworks/seeder.ex +++ b/lib/devils_dictionary/artworks/seeder.ex @@ -1,20 +1,35 @@ defmodule DevilsDictionary.Artworks.Seeder do - @moduledoc "Idempotent, resumable Wikidata-to-Artsy selected artwork importer." + @moduledoc """ + Idempotent, stage-resumable Wikidata-to-Artsy selected artwork importer. + + Wikidata records are immutable observations owned by the shared absorber. This + importer materializes an existing full record when one is available, but it + never replaces that record with a manifest-shaped summary. Artsy payloads are + explicitly disposable caches and every provider-data write is guarded by the + same row lock used by provider withdrawal. + """ + + import Ecto.Query alias DevilsDictionary.Absorb.Materializer alias DevilsDictionary.Absorb.Sources.Wikidata - alias DevilsDictionary.Artsy.Client + alias DevilsDictionary.Artsy.{Availability, Client} alias DevilsDictionary.Artworks.Manifest alias DevilsDictionary.Claims alias DevilsDictionary.Claims.Assertion + alias DevilsDictionary.Corpus.SourceRecordRevision alias DevilsDictionary.Registry + alias DevilsDictionary.Repo alias DevilsDictionary.SourceIdentity alias DevilsDictionary.SourceIdentity.Entry alias DevilsDictionary.Sources + alias DevilsDictionary.Sources.SourceRecord @default_record_limit 500 @max_record_limit 5_000 @default_batch_size 25 + @max_collection_pages 20 + @freshness_seconds 86_400 def run(manifest, opts \\ []) when is_map(manifest) do dry_run? = Keyword.get(opts, :dry_run, false) @@ -33,89 +48,98 @@ defmodule DevilsDictionary.Artworks.Seeder do next_index: 0 }} else - %{sources: sources} = DevilsDictionary.Sources.Catalog.seed!() - artsy_source = Map.fetch!(sources, "artsy") client = opts[:artsy_client] || Client.new(request_limit: opts[:request_limit] || 500) - run = Sources.start_run("dd.artworks.seed", source_id: artsy_source.id) - - initial = %{ - candidates: length(manifest["candidates"]), - processed: 0, - matched: 0, - created: 0, - conflicts: 0, - unavailable: 0, - missing_links: 0, - skipped: 0, - reproductions: 0, - creators_resolved: 0, - creator_links: 0, - genes: 0, - quota_exhausted: false, - requests: 0, - retries: 0, - next_index: nil - } - try do - {manifest, client, summary} = - manifest["candidates"] - |> Enum.with_index() - |> Enum.chunk_every(batch_size) - |> Enum.reduce_while({manifest, client, initial}, fn batch, acc -> - process_batch(batch, acc, sources, run, resume?, record_limit, manifest_path) - end) - - summary = %{ - summary - | requests: client.request_count, - retries: client.retry_count - } - - Sources.finish_run(run, stringify(summary)) - {maybe_save(manifest, manifest_path), summary} - rescue - exception -> - Sources.fail_run(run, Exception.message(exception)) - reraise exception, __STACKTRACE__ + # The importer owns an entry-point gate as well as the client's per-attempt + # gate. An injected/test client must not be able to bypass a withdrawn source, + # and this check happens before Catalog.seed!/0 can touch source lifecycle. + if provider_accessible?(client) and Client.available?(client) do + do_run(manifest, client, resume?, record_limit, batch_size, manifest_path) + else + {manifest, disabled_summary(manifest)} end end end - defp process_batch( - batch, - {manifest, client, summary}, - sources, - run, - resume?, - record_limit, - path - ) do + defp provider_accessible?(%Client{} = client) do + Availability.status_with_credentials( + client.client_id, + client.client_secret, + client.coordinator + ) == :ok + end + + defp do_run(manifest, client, resume?, record_limit, batch_size, manifest_path) do + %{sources: sources} = Sources.Catalog.seed!() + artsy_source = Map.fetch!(sources, "artsy") + run = Sources.start_run("dd.artworks.seed", source_id: artsy_source.id) + + initial = %{ + candidates: length(manifest["candidates"]), + processed: 0, + matched: 0, + created: 0, + conflicts: 0, + unavailable: 0, + missing_links: 0, + skipped: 0, + partial: 0, + reproductions: 0, + creators_resolved: 0, + creator_links: 0, + genes: 0, + request_exhausted: false, + provider_quota_exhausted: false, + quota_exhausted: false, + provider_disabled: false, + requests: 0, + retries: 0, + next_index: nil + } + + try do + {manifest, client, summary} = + manifest["candidates"] + |> Enum.with_index() + |> Enum.chunk_every(batch_size) + |> Enum.reduce_while({manifest, client, initial}, fn batch, acc -> + process_batch(batch, acc, sources, run, resume?, record_limit, manifest_path) + end) + + summary = %{summary | requests: client.request_count, retries: client.retry_count} + Sources.finish_run(run, stringify(summary)) + {maybe_save(manifest, manifest_path), summary} + rescue + exception -> + Sources.fail_run(run, Exception.message(exception)) + reraise exception, __STACKTRACE__ + end + end + + defp process_batch(batch, {manifest, client, summary}, sources, run, resume?, limit, path) do result = - Enum.reduce_while(batch, {manifest, client, summary}, fn {candidate, index}, - {manifest, client, summary} -> + Enum.reduce_while(batch, {manifest, client, summary}, fn {candidate, index}, acc -> + {manifest, client, summary} = acc + cond do - summary.processed >= record_limit -> + summary.processed >= limit -> {:halt, {manifest, client, %{summary | next_index: index}}} resume? and Manifest.completed?(candidate) -> - {:cont, {manifest, client, summary}} + {:cont, acc} true -> {outcome, client} = import_candidate(candidate, client, sources, run) manifest = - Manifest.update_candidate( - manifest, - index, - Map.put(outcome, :updated_at, timestamp()) - ) + manifest + |> Manifest.update_candidate(index, Map.put(outcome, :updated_at, timestamp())) + |> maybe_save(path) summary = accumulate(summary, outcome, index) - manifest = maybe_save(manifest, path) - if outcome.status == :quota_exhausted do - {:halt, {manifest, client, %{summary | quota_exhausted: true, next_index: index}}} + if outcome[:stop] do + {:halt, {manifest, client, summary}} else {:cont, {manifest, client, summary}} end @@ -123,151 +147,180 @@ defmodule DevilsDictionary.Artworks.Seeder do end) case result do - {_manifest, _client, %{quota_exhausted: true}} -> - {:halt, result} - - {_manifest, _client, %{processed: processed}} when processed >= record_limit -> - {:halt, result} - - _ -> - {:cont, result} + {_manifest, _client, %{request_exhausted: true}} -> {:halt, result} + {_manifest, _client, %{provider_quota_exhausted: true}} -> {:halt, result} + {_manifest, _client, %{provider_disabled: true}} -> {:halt, result} + {_manifest, _client, %{processed: processed}} when processed >= limit -> {:halt, result} + _ -> {:cont, result} end end defp import_candidate(candidate, client, sources, run) do - cond do - not eligible_candidate?(candidate) -> - {%{status: :skipped, reason: "invalid_or_unsupported_candidate"}, client} + if eligible_candidate?(candidate) do + existing_id = Registry.by_external_id("wikidata", candidate["qid"]) + {work_id, wikidata_stage} = ensure_wikidata_work!(candidate, sources["wikidata"], run.id) + ensure_wikidata_creator_links(candidate, work_id, sources["wikidata"].id) + + stages = stage_map(candidate) |> Map.put("wikidata", wikidata_stage) + + case cached_artsy_record(candidate, sources["artsy"]) do + %{fresh?: true} = cached -> + continue_hydrated( + candidate, + work_id, + existing_id, + cached, + client, + sources, + run, + Map.put(stages, "artwork", %{"status" => "complete"}) + ) - true -> - existing_id = Registry.by_external_id("wikidata", candidate["qid"]) + %{payload: payload} = cached when is_map(payload) -> + if partial_payload?(payload) do + continue_hydrated( + candidate, + work_id, + existing_id, + cached, + client, + sources, + run, + Map.put(stages, "artwork", %{"status" => "complete"}) + ) + else + fetch_artwork(candidate, work_id, existing_id, client, sources, run, stages) + end - {work_id, _wikidata_record} = - ensure_wikidata_work!(candidate, sources["wikidata"], run.id) + nil -> + fetch_artwork(candidate, work_id, existing_id, client, sources, run, stages) + end + else + {%{status: :skipped, reason: "invalid_or_unsupported_candidate"}, client} + end + end - case Client.artwork(client, candidate["artsy_artwork_slug"]) do - {:ok, body, client, request_meta} -> - artwork = Client.normalize_artwork(body) + defp fetch_artwork(candidate, work_id, existing_id, client, sources, run, stages) do + case Client.artwork(client, candidate["artsy_artwork_slug"]) do + {:ok, body, client, request_meta} -> + artwork = Client.normalize_artwork(body) - import_hydrated( + payload = + artsy_payload(candidate, artwork) + |> put_in(["request_meta"], stringify(request_meta)) + + case guarded_store(payload, sources["artsy"], run.id) do + {:ok, record} -> + cached = %{record: record, payload: payload, fresh?: true} + + continue_hydrated( candidate, - artwork, work_id, existing_id, + cached, client, sources, run, - request_meta + Map.put(stages, "artwork", %{"status" => "complete"}) ) - {:error, %{code: "not_found"}, client} -> - {%{status: :unavailable, object_id: work_id, reason: "artsy_artwork_not_found"}, - client} - - {:error, %{code: "request_limit"}, client} -> - {%{status: :quota_exhausted, object_id: work_id, reason: "request_limit"}, client} + {:error, :provider_disabled} -> + {partial(work_id, stages, "provider_disabled", :provider_disabled), client} + end - {:error, %{code: "quota_exhausted"}, client} -> - {%{status: :quota_exhausted, object_id: work_id, reason: "provider_quota"}, client} + {:error, %{code: "not_found"}, client} -> + {%{ + status: :unavailable, + object_id: work_id, + reason: "artsy_artwork_not_found", + stages: Map.put(stages, "artwork", %{"status" => "unavailable"}) + }, client} - {:error, failure, client} -> - {%{status: :skipped, object_id: work_id, reason: failure.code}, client} - end + {:error, failure, client} -> + {failure_outcome(work_id, stages, "artwork", failure), client} end end - defp import_hydrated( + defp continue_hydrated( candidate, - artwork, work_id, existing_id, + cached, client, sources, run, - request_meta + stages ) do + artwork = cached.payload["artwork"] || %{} + cond do artwork["slug"] != candidate["artsy_artwork_slug"] -> - {record, client} = - store_artsy_record(candidate, artwork, [], [], client, sources["artsy"], run.id) - - {:ok, entry} = - Entry.new(%{ - source_slug: "artsy", - source_id: sources["artsy"].id, - source_record_id: record.id, - source_record_revision_id: record.current_revision.id, - import_run_id: run.id, - object_kind: :entity, - entity_kind: :work, - work_kind: "artwork", - stable_identifier: %{namespace: "wikidata", external_id: candidate["qid"]}, - identifiers: [ - %{namespace: "wikidata", external_id: candidate["qid"]}, - %{namespace: "artsy_artwork_slug", external_id: candidate["artsy_artwork_slug"]}, - %{namespace: "artsy_artwork_slug", external_id: artwork["slug"]}, - %{namespace: "artsy_artwork_id", external_id: artwork["id"]} - ], - label: candidate["title"], - eligibility: :eligible, - retention: :durable - }) - - resolution = SourceIdentity.resolve(entry) + entry = artsy_entry(candidate, artwork, cached.record, sources["artsy"], run.id, true) + resolution = guarded_resolve(entry) {%{ status: :conflict, object_id: work_id, - conflict_id: resolution.conflict_id, + conflict_id: resolution && resolution.conflict_id, reason: "provider_slug_differs_from_wikidata_p11005", requested_slug: candidate["artsy_artwork_slug"], - returned_slug: artwork["slug"] + returned_slug: artwork["slug"], + stages: Map.put(stages, "identity", %{"status" => "conflict"}) }, client} artwork["category"] != "Painting" -> - {record, client} = - store_artsy_record(candidate, artwork, [], [], client, sources["artsy"], run.id) - {%{ status: :skipped, object_id: work_id, - source_record_id: record.id, + source_record_id: cached.record.id, reason: "artsy_category_not_painting", category: artwork["category"], - reproduction: true + reproduction: true, + stages: Map.put(stages, "artwork", %{"status" => "ineligible"}) }, client} true -> - {artists_body, client, artist_failure} = collection(client, :artists, artwork["id"]) - {genes_body, client, gene_failure} = collection(client, :genes, artwork["id"]) + hydrate_collections( + candidate, + work_id, + existing_id, + cached, + client, + sources, + run, + stages + ) + end + end - artists = Enum.map(artists_body, &Client.normalize_artist/1) - genes = Enum.map(genes_body, &Client.normalize_gene/1) + defp hydrate_collections(candidate, work_id, existing_id, cached, client, sources, run, stages) do + with {:ok, cached, client, stages} <- + ensure_collection(:artists, cached, client, sources["artsy"], run.id, stages), + {:ok, cached, client, stages} <- + ensure_collection(:genes, cached, client, sources["artsy"], run.id, stages) do + artists = Enum.map(cached.payload["artists"] || [], &Client.normalize_artist/1) + genes = Enum.map(cached.payload["genes"] || [], &Client.normalize_gene/1) + record = cached.record - {record, client} = - store_artsy_record(candidate, artwork, artists, genes, client, sources["artsy"], run.id) + {:ok, entry} = + artsy_entry(candidate, cached.payload["artwork"], record, sources["artsy"], run.id) - {:ok, entry} = artsy_entry(candidate, artwork, record, sources["artsy"], run.id) - resolution = SourceIdentity.resolve(entry) + case guarded_resolve(entry) do + nil -> + {partial(work_id, stages, "provider_disabled", :provider_disabled), client} - if resolution.state == :conflicting_identifiers do + %{state: :conflicting_identifiers} = resolution -> {%{ status: :conflict, object_id: work_id, conflict_id: resolution.conflict_id, - reason: resolution.reason + reason: resolution.reason, + stages: Map.put(stages, "identity", %{"status" => "conflict"}) }, client} - else - {creator_stats, client} = - import_creators( - candidate, - artists, - resolution.object_id, - record, - client, - sources, - run - ) + + resolution -> + {creator_stats, creator_stage} = + import_creators(candidate, artists, resolution.object_id, record, sources, run) status = if is_nil(existing_id), do: :created, else: :matched @@ -275,63 +328,235 @@ defmodule DevilsDictionary.Artworks.Seeder do status: status, object_id: resolution.object_id, source_record_id: record.id, - artsy_artwork_id: artwork["id"], - artsy_artwork_slug: artwork["slug"], + artsy_artwork_id: cached.payload["artwork"]["id"], + artsy_artwork_slug: cached.payload["artwork"]["slug"], creator_count: creator_stats.resolved, creator_links: creator_stats.links, missing_creators: creator_stats.missing, gene_count: length(genes), - artist_failure: artist_failure, - gene_failure: gene_failure, - redirects: request_meta.redirects + cache_fresh: cached.fresh?, + stages: + stages + |> Map.put("identity", %{"status" => "complete"}) + |> Map.put("creators", creator_stage) }, client} - end + end + else + {:partial, stage, failure, cached, client, stages} -> + outcome = failure_outcome(work_id, stages, to_string(stage), failure) + + outcome = + Map.merge(outcome, %{ + source_record_id: cached.record.id, + retained_artwork: true, + retained_artist_count: length(cached.payload["artists"] || []), + retained_gene_count: length(cached.payload["genes"] || []) + }) + + {outcome, client} end end - defp collection(client, kind, artwork_id) do + defp ensure_collection(kind, cached, client, source, run_id, stages) do + key = if kind == :artists, do: "artists", else: "genes" + state = get_in(cached.payload, ["collection_state", key]) || %{} + + if state["status"] == "complete" do + {:ok, cached, client, Map.put(stages, key, state)} + else + cursor = state["next_cursor"] + fetch_collection(kind, cached, client, source, run_id, stages, cursor, 0) + end + end + + defp fetch_collection(kind, cached, client, source, run_id, stages, cursor, page_count) + when page_count < @max_collection_pages do + artwork_id = cached.payload["artwork"]["id"] + key = if kind == :artists, do: "artists", else: "genes" + result = case kind do - :artists -> Client.artwork_artists(client, artwork_id) - :genes -> Client.artwork_genes(client, artwork_id) + :artists -> Client.artwork_artists(client, artwork_id, cursor: cursor) + :genes -> Client.artwork_genes(client, artwork_id, cursor: cursor) end - key = if kind == :artists, do: "artists", else: "genes" - case result do - {:ok, body, client, _meta} -> {get_in(body, ["_embedded", key]) || [], client, nil} - {:error, failure, client} -> {[], client, failure.code} + {:ok, body, client, meta} -> + rows = get_in(body, ["_embedded", key]) || [] + accumulated = dedupe_source_rows((cached.payload[key] || []) ++ rows) + next_cursor = meta.next_cursor + status = if next_cursor, do: "partial", else: "complete" + + collection_state = %{ + "status" => status, + "next_cursor" => next_cursor, + "pages" => page_count + 1, + "returned_next_preserved_filter" => meta.returned_next_preserved_filter + } + + payload = + cached.payload + |> Map.put(key, accumulated) + |> put_collection_state(key, collection_state) + + case guarded_store(payload, source, run_id) do + {:ok, record} -> + cached = %{cached | payload: payload, record: record} + stages = Map.put(stages, key, collection_state) + + if next_cursor do + fetch_collection( + kind, + cached, + client, + source, + run_id, + stages, + next_cursor, + page_count + 1 + ) + else + {:ok, cached, client, stages} + end + + {:error, :provider_disabled} -> + failure = %{code: "provider_disabled"} + {:partial, kind, failure, cached, client, stages} + end + + {:error, failure, client} -> + state = %{"status" => "partial", "next_cursor" => cursor, "error" => failure.code} + payload = put_collection_state(cached.payload, key, state) + + cached = + case guarded_store(payload, source, run_id) do + {:ok, record} -> %{cached | payload: payload, record: record} + _ -> cached + end + + {:partial, kind, failure, cached, client, Map.put(stages, key, state)} end end - defp store_artsy_record(candidate, artwork, artists, genes, client, source, run_id) do - raw = %{ + defp fetch_collection(kind, cached, client, _source, _run_id, stages, cursor, _page_count) do + failure = %{code: "collection_page_limit"} + state = %{"status" => "partial", "next_cursor" => cursor, "error" => failure.code} + {:partial, kind, failure, cached, client, Map.put(stages, to_string(kind), state)} + end + + defp cached_artsy_record(candidate, source) do + record = + Repo.one( + from record in SourceRecord, + join: revision in SourceRecordRevision, + on: + revision.source_record_id == record.id and + revision.revision_key == record.content_hash, + where: + record.source_id == ^source.id and record.display_allowed and + fragment("?->>'wikidata_qid'", revision.payload) == ^candidate["qid"] and + fragment("?->>'wikidata_p11005'", revision.payload) == + ^candidate["artsy_artwork_slug"], + order_by: [desc: revision.observed_at, desc: revision.id], + limit: 1, + select: + {record, + map(revision, [ + :id, + :source_record_id, + :revision_key, + :payload, + :checksum, + :observed_at, + :import_run_id, + :inserted_at, + :updated_at + ])} + ) + + case record do + {record, revision} -> + age = DateTime.diff(DateTime.utc_now(), revision.observed_at, :second) + + %{ + record: Map.put(record, :current_revision, revision), + payload: revision.payload, + fresh?: age <= freshness_seconds() + } + + nil -> + nil + end + end + + defp guarded_store(payload, source, run_id) do + case Availability.with_active_source(fn _locked_source -> + artwork = payload["artwork"] + + Sources.upsert_record(source, %{ + external_id: "artwork:#{artwork["id"]}", + url: artwork["permalink"], + raw: payload, + import_run_id: run_id + }) + end) do + {:ok, {:ok, record}} -> {:ok, record} + {:error, :provider_disabled} -> {:error, :provider_disabled} + {:error, reason} -> raise "Artsy record write failed: #{inspect(reason)}" + end + end + + defp guarded_resolve({:ok, entry}), do: guarded_resolve(entry) + + defp guarded_resolve(%Entry{} = entry) do + case Availability.with_active_source(fn _ -> SourceIdentity.resolve(entry) end) do + {:ok, resolution} -> resolution + {:error, :provider_disabled} -> nil + end + end + + defp artsy_payload(candidate, artwork) do + %{ "artwork" => artwork, - "artists" => artists, - "genes" => genes, + "artists" => [], + "genes" => [], + "collection_state" => %{ + "artists" => %{"status" => "pending", "next_cursor" => nil}, + "genes" => %{"status" => "pending", "next_cursor" => nil} + }, "wikidata_qid" => candidate["qid"], "wikidata_p11005" => candidate["artsy_artwork_slug"], "selection_reason" => candidate["selection_reason"], + "freshness" => %{ + "observed_at" => timestamp(), + "fresh_for_seconds" => freshness_seconds(), + "retained_content_may_be_stale" => true + }, "retention" => %{ - "mode" => "discovery_cache", - "image" => "reference_only", + "mode" => "disposable_discovery_cache", + "image" => "remote_reference_only", "durable_source" => "wikidata", - "remove_on_provider_termination" => true + "remove_on_provider_termination" => true, + "api_access_does_not_grant_permanent_retention" => true } } - - {:ok, record} = - Sources.upsert_record(source, %{ - external_id: "artwork:#{artwork["id"]}", - url: artwork["permalink"], - raw: raw, - import_run_id: run_id - }) - - {record, client} end - defp artsy_entry(candidate, artwork, record, source, run_id) do + defp artsy_entry(candidate, artwork, record, source, run_id, conflict? \\ false) do + identifiers = [ + %{namespace: "wikidata", external_id: candidate["qid"]}, + %{namespace: "artsy_artwork_slug", external_id: artwork["slug"]}, + %{namespace: "artsy_artwork_id", external_id: artwork["id"]} + ] + + identifiers = + if conflict?, + do: [ + %{namespace: "artsy_artwork_slug", external_id: candidate["artsy_artwork_slug"]} + | identifiers + ], + else: identifiers + Entry.new(%{ source_slug: source.slug, source_id: source.id, @@ -342,198 +567,348 @@ defmodule DevilsDictionary.Artworks.Seeder do entity_kind: :work, work_kind: "artwork", stable_identifier: %{namespace: "artsy_artwork_id", external_id: artwork["id"]}, + identifiers: identifiers, + label: candidate["title"] || artwork["title"], + metadata: %{"provider" => "artsy", "retention" => "disposable_cache"}, + eligibility: :eligible, + retention: :durable + }) + end + + defp ensure_wikidata_work!(candidate, source, run_id) do + case current_source_record(source.id, candidate["qid"]) do + nil -> + {:ok, entry} = wikidata_work_entry(candidate, source) + resolution = SourceIdentity.resolve(entry) + {resolution.object_id, %{"status" => "exact_manifest_link", "source_record_id" => nil}} + + {record, revision} -> + record = record |> Map.put(:raw, revision.payload) |> Map.put(:current_revision, revision) + {:ok, _} = Materializer.run(record, Wikidata, run_id: run_id) + + work_id = + Registry.by_external_id("wikidata", candidate["qid"]) || + resolve_wikidata_work(candidate, source, record, revision, run_id) + + {work_id, + %{ + "status" => "materialized_full_record", + "source_record_id" => record.id, + "source_record_revision_id" => revision.id, + "preserved" => true + }} + end + end + + defp wikidata_work_entry(candidate, source) do + Entry.new(%{ + source_slug: source.slug, + source_id: source.id, + object_kind: :entity, + entity_kind: :work, + work_kind: "artwork", + stable_identifier: %{namespace: "wikidata", external_id: candidate["qid"]}, identifiers: [ %{namespace: "wikidata", external_id: candidate["qid"]}, - %{namespace: "artsy_artwork_slug", external_id: artwork["slug"]}, - %{namespace: "artsy_artwork_id", external_id: artwork["id"]} + %{namespace: "artsy_artwork_slug", external_id: candidate["artsy_artwork_slug"]} ], label: candidate["title"], - metadata: %{}, + description: nil, + metadata: %{ + "identity_evidence" => "wikidata_exact_qid_p11005_manifest", + "not_a_wikidata_record" => true + }, eligibility: :eligible, retention: :durable }) end - defp import_creators(candidate, artists, work_id, artwork_record, client, sources, run) do - by_slug = Map.new(candidate["creators"] || [], &{&1["artsy_artist_slug"], &1}) - - Enum.reduce(artists, {%{resolved: 0, links: 0, missing: 0}, client}, fn artist, - {stats, client} -> - case by_slug[artist["slug"]] do - %{"qid" => qid} = creator when is_binary(qid) -> - {person_id, _record} = ensure_wikidata_creator!(creator, sources["wikidata"], run.id) - - {:ok, artist_record} = - Sources.upsert_record(sources["artsy"], %{ - external_id: "artist:#{artist["id"]}", - url: artist["permalink"], - raw: %{ - "artist" => artist, - "wikidata_qid" => qid, - "wikidata_p2042" => creator["artsy_artist_slug"], - "retention" => %{"remove_on_provider_termination" => true} - }, - import_run_id: run.id - }) - - {:ok, entry} = - Entry.new(%{ - source_slug: "artsy", - source_id: sources["artsy"].id, - source_record_id: artist_record.id, - source_record_revision_id: artist_record.current_revision.id, - import_run_id: run.id, - object_kind: :entity, - entity_kind: :person, - stable_identifier: %{namespace: "artsy_artist_id", external_id: artist["id"]}, - identifiers: [ - %{namespace: "wikidata", external_id: qid}, - %{namespace: "artsy_artist_slug", external_id: artist["slug"]}, - %{namespace: "artsy_artist_id", external_id: artist["id"]} - ], - label: creator["name"] || artist["name"], - metadata: %{}, - eligibility: :eligible, - retention: :durable - }) - - resolution = SourceIdentity.resolve(entry) - - if resolution.object_id == person_id do - linked? = ensure_creator_link(work_id, person_id, sources["artsy"].id, artwork_record) - - {%{ - stats - | resolved: stats.resolved + 1, - links: stats.links + if(linked?, do: 1, else: 0) - }, client} - else - {%{stats | missing: stats.missing + 1}, client} - end + defp resolve_wikidata_work(candidate, source, record, revision, run_id) do + {:ok, entry} = wikidata_work_entry(candidate, source) + + entry = %{ + entry + | source_record_id: record.id, + source_record_revision_id: revision.id, + import_run_id: run_id + } + + SourceIdentity.resolve(entry).object_id + end + + defp ensure_wikidata_creator_links(candidate, work_id, source_id) do + Enum.each(candidate["creators"] || [], fn creator -> + if valid_qid?(creator["qid"]) do + person_id = ensure_wikidata_creator!(creator, source_id) - _ -> - {%{stats | missing: stats.missing + 1}, client} + if is_integer(work_id) and is_integer(person_id) do + ensure_creator_link(work_id, person_id, source_id, nil, "wikidata_manifest") + end end end) end - defp ensure_creator_link(work_id, person_id, source_id, artwork_record) do - origin_key = "artsy:creator:#{work_id}:#{person_id}" + defp ensure_wikidata_creator!(creator, source_id) do + identifiers = + [%{namespace: "wikidata", external_id: creator["qid"]}] ++ + if(valid_slug?(creator["artsy_artist_slug"]), + do: [%{namespace: "artsy_artist_slug", external_id: creator["artsy_artist_slug"]}], + else: [] + ) - case DevilsDictionary.Repo.get_by(Assertion, source_id: source_id, origin_key: origin_key) do - nil -> - claim = - unwrap( - Claims.assert(work_id, "authored_by", person_id, %{ - source_id: source_id, - origin_key: origin_key, - method: "source_relationship", - metadata: %{"provider" => "artsy", "review" => "attributed_source_fact"} - }) - ) + {:ok, entry} = + Entry.new(%{ + source_slug: "wikidata", + source_id: source_id, + object_kind: :entity, + entity_kind: :person, + stable_identifier: %{namespace: "wikidata", external_id: creator["qid"]}, + identifiers: identifiers, + label: creator["name"] || creator["qid"], + metadata: %{ + "identity_evidence" => "wikidata_exact_p170_manifest", + "not_a_wikidata_record" => true + }, + eligibility: :eligible, + retention: :durable + }) - revision = Claims.current_revision(claim.id) + SourceIdentity.resolve(entry).object_id + end - unwrap( - Claims.add_evidence(revision.id, %{ - source_record_revision_id: artwork_record.current_revision.id, - evidence_role: :supports, - attribution_text: "Artsy artwork creator link" - }) - ) + defp import_creators(candidate, artists, work_id, artwork_record, sources, run) do + creator_by_slug = + (candidate["creators"] || []) + |> Enum.filter(&valid_slug?(&1["artsy_artist_slug"])) + |> Map.new(&{&1["artsy_artist_slug"], &1}) + + stats = + Enum.reduce(artists, %{resolved: 0, links: 0, missing: 0}, fn artist, stats -> + case creator_by_slug[artist["slug"]] do + %{"qid" => qid} = creator when is_binary(qid) -> + person_id = Registry.by_external_id("wikidata", qid) + + if person_id do + case guarded_store_artist(artist, creator, sources["artsy"], run.id) do + {:ok, artist_record} -> + {:ok, entry} = + artsy_artist_entry(artist, creator, artist_record, sources["artsy"], run.id) + + resolution = guarded_resolve(entry) + + if resolution && resolution.object_id == person_id do + linked? = + ensure_creator_link( + work_id, + person_id, + sources["artsy"].id, + artwork_record, + "artsy" + ) + + %{ + stats + | resolved: stats.resolved + 1, + links: stats.links + if(linked?, do: 1, else: 0) + } + else + %{stats | missing: stats.missing + 1} + end + + {:error, :provider_disabled} -> + %{stats | missing: stats.missing + 1} + end + else + %{stats | missing: stats.missing + 1} + end - true + _ -> + %{stats | missing: stats.missing + 1} + end + end) - _existing -> - false - end + {stats, %{"status" => "complete", "resolved" => stats.resolved, "missing" => stats.missing}} end - defp ensure_wikidata_work!(candidate, source, run_id) do - raw = wikidata_snapshot(candidate, :artwork) + defp guarded_store_artist(artist, creator, source, run_id) do + payload = %{ + "artist" => artist, + "wikidata_qid" => creator["qid"], + "wikidata_p2042" => creator["artsy_artist_slug"], + "retention" => %{"remove_on_provider_termination" => true} + } - {:ok, record} = - Sources.upsert_record(source, %{ - external_id: candidate["qid"], - url: "https://www.wikidata.org/wiki/#{candidate["qid"]}", - raw: raw, - import_run_id: run_id - }) + case Availability.with_active_source(fn _ -> + Sources.upsert_record(source, %{ + external_id: "artist:#{artist["id"]}", + url: artist["permalink"], + raw: payload, + import_run_id: run_id + }) + end) do + {:ok, {:ok, record}} -> {:ok, record} + {:error, :provider_disabled} -> {:error, :provider_disabled} + end + end - {:ok, _} = Materializer.run(record, Wikidata, run_id: run_id) - {Registry.by_external_id("wikidata", candidate["qid"]), record} + defp artsy_artist_entry(artist, creator, record, source, run_id) do + Entry.new(%{ + source_slug: source.slug, + source_id: source.id, + source_record_id: record.id, + source_record_revision_id: record.current_revision.id, + import_run_id: run_id, + object_kind: :entity, + entity_kind: :person, + stable_identifier: %{namespace: "artsy_artist_id", external_id: artist["id"]}, + identifiers: [ + %{namespace: "wikidata", external_id: creator["qid"]}, + %{namespace: "artsy_artist_slug", external_id: artist["slug"]}, + %{namespace: "artsy_artist_id", external_id: artist["id"]} + ], + label: creator["name"] || artist["name"], + metadata: %{"provider" => "artsy", "retention" => "disposable_cache"}, + eligibility: :eligible, + retention: :durable + }) end - defp ensure_wikidata_creator!(creator, source, run_id) do - raw = wikidata_snapshot(creator, :person) + defp ensure_creator_link(work_id, person_id, source_id, artwork_record, provider) do + origin_key = "#{provider}:creator:#{work_id}:#{person_id}" - {:ok, record} = - Sources.upsert_record(source, %{ - external_id: creator["qid"], - url: "https://www.wikidata.org/wiki/#{creator["qid"]}", - raw: raw, - import_run_id: run_id - }) + case Repo.get_by(Assertion, source_id: source_id, origin_key: origin_key) do + nil -> + case Claims.assert(work_id, "authored_by", person_id, %{ + source_id: source_id, + origin_key: origin_key, + method: "source_relationship", + metadata: %{ + "provider" => provider, + "review" => "attributed_source_fact", + "identity_independent_of_artwork_provider" => provider == "wikidata_manifest" + } + }) do + {:ok, claim} -> + if artwork_record do + revision = Claims.current_revision(claim.id) + + {:ok, _evidence} = + Claims.add_evidence(revision.id, %{ + source_record_revision_id: artwork_record.current_revision.id, + evidence_role: :supports, + attribution_text: "Artsy artwork creator link" + }) + end - {:ok, _} = Materializer.run(record, Wikidata, run_id: run_id) - {Registry.by_external_id("wikidata", creator["qid"]), record} - end + true - defp wikidata_snapshot(value, kind) do - claims = - case kind do - :artwork -> - %{ - "P31" => [entity_statement("Q3305213")], - "P11005" => [string_statement(value["artsy_artwork_slug"])], - "P170" => Enum.map(value["creators"] || [], &entity_statement(&1["qid"])) - } - - :person -> - %{ - "P31" => [entity_statement("Q5")], - "P2042" => [string_statement(value["artsy_artist_slug"])] - } - end + {:error, _reason} -> + false + end - %{ - "_artwork_seed_version" => 1, - "id" => value["qid"], - "labels" => %{"en" => %{"language" => "en", "value" => value["title"] || value["name"]}}, - "descriptions" => description(value["description"]), - "aliases" => %{}, - "sitelinks" => sitelink(value["wikipedia_title"]), - "claims" => claims - } + _existing -> + false + end end - defp entity_statement(nil), do: %{} + defp current_source_record(source_id, external_id) do + Repo.one( + from record in SourceRecord, + join: revision in SourceRecordRevision, + on: + revision.source_record_id == record.id and revision.revision_key == record.content_hash, + where: record.source_id == ^source_id and record.external_id == ^external_id, + select: + {record, + map(revision, [ + :id, + :source_record_id, + :revision_key, + :payload, + :checksum, + :observed_at, + :import_run_id, + :inserted_at, + :updated_at + ])} + ) + end - defp entity_statement(qid) do - %{ - "type" => "statement", - "rank" => "normal", - "mainsnak" => %{"datavalue" => %{"value" => %{"id" => qid}, "type" => "wikibase-entityid"}} - } + defp stage_map(candidate) do + get_in(candidate, ["import", "stages"]) || + %{ + "wikidata" => %{"status" => "pending"}, + "artwork" => %{"status" => "pending"}, + "artists" => %{"status" => "pending", "next_cursor" => nil}, + "genes" => %{"status" => "pending", "next_cursor" => nil}, + "identity" => %{"status" => "pending"}, + "creators" => %{"status" => "pending"} + } + end + + defp failure_outcome(work_id, stages, stage, failure) do + status = + if failure.code in [ + "request_limit", + "quota_exhausted", + "provider_disabled", + "collection_page_limit" + ], do: :partial, else: :partial + + partial( + work_id, + Map.put(stages, stage, %{"status" => "partial", "error" => failure.code}), + failure.code, + failure_kind(failure.code) + ) + |> Map.put(:status, status) end - defp string_statement(value) do + defp partial(work_id, stages, reason, kind) do %{ - "type" => "statement", - "rank" => "normal", - "mainsnak" => %{"datavalue" => %{"value" => value, "type" => "string"}} + status: :partial, + object_id: work_id, + reason: reason, + failure_kind: kind, + stages: stages, + stop: kind in [:request_exhausted, :provider_quota, :provider_disabled] } end - defp description(nil), do: %{} - defp description(value), do: %{"en" => %{"language" => "en", "value" => value}} - defp sitelink(nil), do: %{} - defp sitelink(title), do: %{"enwiki" => %{"site" => "enwiki", "title" => title}} + defp failure_kind("request_limit"), do: :request_exhausted + defp failure_kind("quota_exhausted"), do: :provider_quota + defp failure_kind("provider_disabled"), do: :provider_disabled + defp failure_kind(_), do: :transient defp eligible_candidate?(candidate) do - candidate["kind"] == "painting" and Regex.match?(~r/\AQ[1-9]\d*\z/, candidate["qid"] || "") and - Regex.match?(~r/\A[a-z0-9][a-z0-9-]{1,254}\z/, candidate["artsy_artwork_slug"] || "") and - is_binary(candidate["title"]) and String.trim(candidate["title"]) != "" + candidate["kind"] == "painting" and valid_qid?(candidate["qid"]) and + valid_slug?(candidate["artsy_artwork_slug"]) and is_binary(candidate["title"]) and + String.trim(candidate["title"]) != "" + end + + defp valid_qid?(value), do: is_binary(value) and Regex.match?(~r/\AQ[1-9]\d*\z/, value) + + defp valid_slug?(value), + do: is_binary(value) and Regex.match?(~r/\A[a-z0-9][a-z0-9-]{1,254}\z/, value) + + defp dedupe_source_rows(rows) do + Enum.uniq_by(rows, fn row -> row["id"] || row["slug"] || :erlang.phash2(row) end) + end + + defp partial_payload?(payload) do + Enum.any?(~w(artists genes), fn key -> + get_in(payload, ["collection_state", key, "status"]) != "complete" + end) + end + + defp put_collection_state(payload, key, state) do + Map.update(payload, "collection_state", %{key => state}, fn collection_state -> + Map.put(collection_state || %{}, key, state) + end) + end + + defp freshness_seconds do + Application.get_env(:devils_dictionary, :artsy, [])[:freshness_seconds] || @freshness_seconds end defp accumulate(summary, outcome, index) do @@ -545,11 +920,15 @@ defmodule DevilsDictionary.Artworks.Seeder do :matched -> %{summary | matched: summary.matched + 1} :conflict -> %{summary | conflicts: summary.conflicts + 1} :unavailable -> %{summary | unavailable: summary.unavailable + 1} - :quota_exhausted -> summary + :partial -> %{summary | partial: summary.partial + 1} :skipped -> %{summary | skipped: summary.skipped + 1} end summary + |> Map.put(:request_exhausted, outcome[:failure_kind] == :request_exhausted) + |> Map.put(:provider_quota_exhausted, outcome[:failure_kind] == :provider_quota) + |> Map.put(:quota_exhausted, outcome[:failure_kind] in [:request_exhausted, :provider_quota]) + |> Map.put(:provider_disabled, outcome[:failure_kind] == :provider_disabled) |> Map.update!(:reproductions, &(&1 + if(outcome[:reproduction], do: 1, else: 0))) |> Map.update!(:creators_resolved, &(&1 + (outcome[:creator_count] || 0))) |> Map.update!(:creator_links, &(&1 + (outcome[:creator_links] || 0))) @@ -557,11 +936,40 @@ defmodule DevilsDictionary.Artworks.Seeder do |> Map.update!(:genes, &(&1 + (outcome[:gene_count] || 0))) end + defp disabled_summary(manifest) do + %{ + candidates: length(manifest["candidates"]), + processed: 0, + matched: 0, + created: 0, + conflicts: 0, + unavailable: 0, + missing_links: 0, + skipped: 0, + partial: 0, + reproductions: 0, + creators_resolved: 0, + creator_links: 0, + genes: 0, + request_exhausted: false, + provider_quota_exhausted: false, + quota_exhausted: false, + provider_disabled: true, + requests: 0, + retries: 0, + next_index: 0 + } + end + defp maybe_save(manifest, nil), do: manifest defp maybe_save(manifest, path), do: Manifest.save!(manifest, path) defp bounded(value, min, max) when is_integer(value), do: value |> max(min) |> min(max) - defp stringify(map), do: Map.new(map, fn {key, value} -> {to_string(key), value} end) + + defp stringify(map) when is_map(map), + do: Map.new(map, fn {key, value} -> {to_string(key), stringify(value)} end) + + defp stringify(value) when is_list(value), do: Enum.map(value, &stringify/1) + defp stringify(value) when is_atom(value), do: to_string(value) + defp stringify(value), do: value defp timestamp, do: DateTime.utc_now() |> DateTime.truncate(:second) |> DateTime.to_iso8601() - defp unwrap({:ok, value}), do: value - defp unwrap({:error, reason}), do: DevilsDictionary.Repo.rollback(reason) end diff --git a/lib/devils_dictionary/artworks/wikidata_candidates.ex b/lib/devils_dictionary/artworks/wikidata_candidates.ex index 8acc1b77..9bf11e88 100644 --- a/lib/devils_dictionary/artworks/wikidata_candidates.ex +++ b/lib/devils_dictionary/artworks/wikidata_candidates.ex @@ -3,6 +3,8 @@ defmodule DevilsDictionary.Artworks.WikidataCandidates do import Ecto.Query + alias DevilsDictionary.Claims + alias DevilsDictionary.Claims.AssertionRevision alias DevilsDictionary.Registry.{Entity, ExternalIdentifier, WorkDetails} alias DevilsDictionary.Repo @@ -31,15 +33,19 @@ defmodule DevilsDictionary.Artworks.WikidataCandidates do {[], %{requests: 0, next_offset: offset, truncated: remaining > 0}} end - candidates = - (local ++ remote) - |> Enum.uniq_by(&{&1["qid"], &1["artsy_artwork_slug"]}) - |> Enum.take(limit) + candidates = merge_candidates(local ++ remote) |> Enum.take(limit) + local_identities = MapSet.new(local, &{&1["qid"], &1["artsy_artwork_slug"]}) + + remote_count = + Enum.count( + candidates, + &(!MapSet.member?(local_identities, {&1["qid"], &1["artsy_artwork_slug"]})) + ) {:ok, candidates, Map.merge(stats, %{ local: length(local), - remote: length(candidates) - length(local), + remote: remote_count, discovered: length(candidates), limit: limit, kinds: ["painting"] @@ -47,31 +53,85 @@ defmodule DevilsDictionary.Artworks.WikidataCandidates do end defp local_candidates(limit) do - Repo.all( - from entity in Entity, - join: details in WorkDetails, - on: details.entity_id == entity.object_id and details.work_kind == "artwork", - join: artsy in ExternalIdentifier, - on: - artsy.object_id == entity.object_id and artsy.namespace == "artsy_artwork_slug" and - artsy.status == :verified, - join: qid in ExternalIdentifier, - on: - qid.object_id == entity.object_id and qid.namespace == "wikidata" and - qid.status == :verified, - order_by: [asc: entity.object_id], - limit: ^limit, - select: %{ - "qid" => qid.external_id, - "artsy_artwork_slug" => artsy.external_id, - "title" => entity.preferred_label, - "description" => entity.description, - "wikipedia_title" => entity.metadata["wikipedia_title"], - "kind" => "painting", - "selection_reason" => "existing_local_wikidata_p11005", - "creators" => [] - } + rows = + Repo.all( + from entity in Entity, + join: details in WorkDetails, + on: details.entity_id == entity.object_id and details.work_kind == "artwork", + join: artsy in ExternalIdentifier, + on: + artsy.object_id == entity.object_id and artsy.namespace == "artsy_artwork_slug" and + artsy.status == :verified, + join: qid in ExternalIdentifier, + on: + qid.object_id == entity.object_id and qid.namespace == "wikidata" and + qid.status == :verified, + order_by: [asc: entity.object_id], + limit: ^limit, + select: %{ + "object_id" => entity.object_id, + "qid" => qid.external_id, + "artsy_artwork_slug" => artsy.external_id, + "title" => entity.preferred_label, + "description" => entity.description, + "wikipedia_title" => entity.metadata["wikipedia_title"], + "kind" => "painting", + "selection_reason" => "existing_local_wikidata_p11005", + "origin" => "local_registry" + } + ) + + creators = local_creators(Enum.map(rows, & &1["object_id"])) + + Enum.map(rows, fn row -> + row + |> Map.put("creators", Map.get(creators, row["object_id"], [])) + |> Map.delete("object_id") + end) + end + + defp local_creators([]), do: %{} + + defp local_creators(work_ids) do + AssertionRevision + |> join(:inner, [revision], predicate in Claims.Predicate, + on: predicate.id == revision.predicate_id and predicate.key == "authored_by" + ) + |> join(:inner, [revision], creator in Entity, + on: creator.object_id == revision.object_object_id + ) + |> join(:inner, [_revision, _predicate, creator], qid in ExternalIdentifier, + on: + qid.object_id == creator.object_id and qid.namespace == "wikidata" and + qid.status == :verified + ) + |> join(:left, [_revision, _predicate, creator, _qid], artsy in ExternalIdentifier, + on: + artsy.object_id == creator.object_id and artsy.namespace == "artsy_artist_slug" and + artsy.status == :verified ) + |> where( + [revision], + revision.subject_object_id in ^work_ids and revision.is_current and + revision.lifecycle_state == :active + ) + |> Claims.visible(:public) + |> order_by( + [revision, _predicate, creator, qid, artsy], + asc: revision.subject_object_id, + asc: creator.preferred_label, + asc: creator.object_id + ) + |> select([revision, _predicate, creator, qid, artsy], { + revision.subject_object_id, + %{ + "qid" => qid.external_id, + "name" => creator.preferred_label, + "artsy_artist_slug" => artsy.external_id + } + }) + |> Repo.all() + |> Enum.group_by(&elem(&1, 0), &elem(&1, 1)) end defp remote_candidates(limit, offset, page_size, request_limit, request_fun) do @@ -79,13 +139,12 @@ defmodule DevilsDictionary.Artworks.WikidataCandidates do end defp do_remote(rows, limit, offset, _page_size, request_limit, requests, _request_fun) - when length(rows) >= limit or requests >= request_limit do - {Enum.take(rows, limit), - %{requests: requests, next_offset: offset, truncated: length(rows) < limit}} + when length(rows) > limit or requests >= request_limit do + {Enum.take(rows, limit), %{requests: requests, next_offset: offset, truncated: true}} end defp do_remote(rows, limit, offset, page_size, request_limit, requests, request_fun) do - take = min(page_size, max(limit - length(rows), 1)) + take = page_size case request_fun.(request_options(take, offset)) do {:ok, %Req.Response{status: 200, body: body}} -> @@ -98,7 +157,7 @@ defmodule DevilsDictionary.Artworks.WikidataCandidates do %{requests: requests + 1, next_offset: next_offset, truncated: false}} else do_remote( - dedupe(rows ++ page), + merge_candidates(rows ++ page), limit, next_offset, page_size, @@ -181,6 +240,7 @@ defmodule DevilsDictionary.Artworks.WikidataCandidates do do: "wikidata_p11005_with_english_wikipedia", else: "wikidata_p11005_painting" ), + "origin" => "wikidata_sparql", "creators" => rows |> Enum.map(fn row -> @@ -197,7 +257,33 @@ defmodule DevilsDictionary.Artworks.WikidataCandidates do |> Enum.sort_by(& &1["qid"]) end - defp dedupe(rows), do: Enum.uniq_by(rows, &{&1["qid"], &1["artsy_artwork_slug"]}) + defp merge_candidates(rows) do + rows + |> Enum.group_by(&{&1["qid"], &1["artsy_artwork_slug"]}) + |> Enum.map(fn {_identity, observations} -> + preferred = + Enum.find(observations, &(&1["origin"] == "wikidata_sparql")) || hd(observations) + + creators = + observations + |> Enum.flat_map(&(&1["creators"] || [])) + |> Enum.group_by(& &1["qid"]) + |> Enum.map(fn {_qid, rows} -> + Enum.reduce(rows, %{}, fn row, merged -> + Map.merge(merged, row, fn _key, held, offered -> held || offered end) + end) + end) + |> Enum.sort_by(&{&1["name"] || "", &1["qid"] || ""}) + + observations + |> Enum.reduce(preferred, fn row, merged -> + Map.merge(merged, row, fn _key, held, offered -> held || offered end) + end) + |> Map.put("creators", creators) + end) + |> Enum.sort_by(&{&1["qid"], &1["artsy_artwork_slug"]}) + end + defp value(row, key), do: get_in(row, [key, "value"]) defp qid("http://www.wikidata.org/entity/" <> qid), do: if(valid_qid?(qid), do: qid) diff --git a/lib/devils_dictionary_web/components/artwork.ex b/lib/devils_dictionary_web/components/artwork.ex index b90d9c7a..03b8525a 100644 --- a/lib/devils_dictionary_web/components/artwork.ex +++ b/lib/devils_dictionary_web/components/artwork.ex @@ -59,6 +59,23 @@ defmodule DevilsDictionaryWeb.Artwork do > {@artwork.description}

+
+
+
Date
{@artwork.date}
+
+
+
Medium
· {@artwork.medium}
+
+
+
Collection
· {@artwork.collection}
+
+
+

+ Image: {@artwork.image_attribution} +

@@ -66,7 +83,10 @@ defmodule DevilsDictionaryWeb.Artwork do ({@candidate.language})

- Direct Artsy gene “{@candidate.match_reason.gene_name}” · not yet reviewed + {candidate_relation(@candidate.match_type)} Artsy gene “{@candidate.match_reason.gene_name}” · not yet reviewed +

+

+ {@candidate.match_reason.note}

@@ -74,7 +94,7 @@ defmodule DevilsDictionaryWeb.Artwork do <.link navigate={entity_path(@artwork)} class="font-medium underline underline-offset-4">Open work <.link :if={@connect} - navigate={~p"/connect?subject=#{@artwork.object_id}"} + navigate={connect_path(@artwork, @candidate)} class="font-medium underline underline-offset-4" > Connect to a meaning @@ -97,4 +117,15 @@ defmodule DevilsDictionaryWeb.Artwork do defp creator_path(creator), do: ~p"/entities/#{creator.object_id}/#{Connection.slugify(creator.label)}" + + defp connect_path(artwork, nil), do: ~p"/connect?subject=#{artwork.object_id}" + + defp connect_path(artwork, candidate) do + ~p"/connect?#{%{subject: artwork.object_id, object: candidate.sense_id, predicate: "illustrates", evidence_revision: candidate.source_record_revision_id, evidence_locator: "Artsy direct gene #{candidate.match_reason.gene_id}", rationale: candidate.match_reason.note}}" + end + + defp candidate_relation(:broader), do: "Broader-context" + defp candidate_relation(:related), do: "Related" + defp candidate_relation("broader"), do: "Broader-context" + defp candidate_relation(_), do: "Direct" end diff --git a/lib/devils_dictionary_web/live/artwork_live.ex b/lib/devils_dictionary_web/live/artwork_live.ex index 304fdcba..2944dc62 100644 --- a/lib/devils_dictionary_web/live/artwork_live.ex +++ b/lib/devils_dictionary_web/live/artwork_live.ex @@ -13,7 +13,9 @@ defmodule DevilsDictionaryWeb.ArtworkLive do @impl true def mount(_params, _session, socket) do - configured? = Client.new() |> Client.configured?() + client = Client.new() + available? = Client.available?(client) + catalog = Artworks.catalog("") {:ok, socket @@ -21,29 +23,58 @@ defmodule DevilsDictionaryWeb.ArtworkLive do page_title: "artworks", search_form: to_form(%{"q" => ""}, as: :search), artsy_form: to_form(%{"q" => ""}, as: :artsy), - artsy_configured: configured?, + artsy_configured: available?, artsy_requests_used: 0, artsy_status: :idle, artsy_items: [], artsy_query: "", - contributor: Contributions.internal_contributor?(socket.assigns[:current_scope]) + contributor: Contributions.internal_contributor?(socket.assigns[:current_scope]), + catalog_query: "", + catalog_page: catalog.page, + catalog_count: catalog.count, + catalog_previous_page: catalog.previous_page, + catalog_next_page: catalog.next_page ) - |> stream(:artworks, Artworks.search(""), dom_id: &"artwork-#{&1.object_id}")} + |> stream(:artworks, catalog.items, dom_id: &"artwork-#{&1.object_id}")} end @impl true def handle_event("search-local", %{"search" => %{"q" => query}}, socket) do + catalog = Artworks.catalog(query) + + {:noreply, + socket + |> assign( + search_form: to_form(%{"q" => query}, as: :search), + catalog_query: query, + catalog_page: catalog.page, + catalog_count: catalog.count, + catalog_previous_page: catalog.previous_page, + catalog_next_page: catalog.next_page + ) + |> stream(:artworks, catalog.items, reset: true)} + end + + def handle_event("catalog-page", %{"page" => page}, socket) do + page = parse_page(page) + catalog = Artworks.catalog(socket.assigns.catalog_query, page: page) + {:noreply, socket - |> assign(:search_form, to_form(%{"q" => query}, as: :search)) - |> stream(:artworks, Artworks.search(query), reset: true)} + |> assign( + catalog_page: catalog.page, + catalog_count: catalog.count, + catalog_previous_page: catalog.previous_page, + catalog_next_page: catalog.next_page + ) + |> stream(:artworks, catalog.items, reset: true)} end def handle_event("search-artsy", %{"artsy" => %{"q" => query}}, socket) do query = String.trim(query) cond do - not socket.assigns.artsy_configured -> + not Client.available?(Client.new()) -> {:noreply, assign(socket, artsy_status: :disabled, artsy_items: [])} query == "" -> @@ -138,6 +169,9 @@ defmodule DevilsDictionaryWeb.ArtworkLive do
+

+ {@catalog_count} saved {if @catalog_count == 1, do: "artwork", else: "artworks"} +

+

Already saved · exact P11005 match

<% else %> -

{item.search["title"] || "Untitled result"}

-

{artsy_result_label(item.status)}

- Open source ↗ +
+
+ +
+
+

{item.search["title"] || "Untitled result"}

+

{artsy_result_label(item.status)}

+ Open source ↗ +
+
<% end %> @@ -249,4 +324,15 @@ defmodule DevilsDictionaryWeb.ArtworkLive do defp artsy_result_label(:unsupported_type), do: "Unsupported non-artwork result" defp artsy_result_label(:invalid_endpoint), do: "Ignored unsafe provider endpoint" defp artsy_result_label(_), do: "Source lookup failed for this result" + + defp parse_page(value) when is_integer(value) and value > 0, do: value + + defp parse_page(value) when is_binary(value) do + case Integer.parse(value) do + {page, ""} when page > 0 -> page + _ -> 1 + end + end + + defp parse_page(_), do: 1 end diff --git a/lib/devils_dictionary_web/live/connection_live.ex b/lib/devils_dictionary_web/live/connection_live.ex index 7da131b5..6ff7b81f 100644 --- a/lib/devils_dictionary_web/live/connection_live.ex +++ b/lib/devils_dictionary_web/live/connection_live.ex @@ -169,12 +169,47 @@ defmodule DevilsDictionaryWeb.ConnectionLive do def handle_params(params, _uri, socket) do subject = preselected_subject(params["subject"]) + object = preselected_subject(params["object"]) + predicates = predicates_for(subject) + + predicate = + Enum.find_value(predicates, fn option -> + if option.key == params["predicate"], do: option.key + end) + + object = + if object && predicate && valid_object_hits([object], subject, predicate) != [], + do: object, + else: nil + + rationale = String.trim(params["rationale"] || "") + locator = String.trim(params["evidence_locator"] || "") + + {evidence_items, evidence_rows} = + preselected_evidence(params["evidence_revision"], locator) + {:noreply, - assign(socket, + socket + |> assign( page_title: "propose a connection", subject: subject, - predicates: predicates_for(subject) - )} + object: object, + predicate: predicate, + predicates: predicates, + rationale: rationale, + locator: locator, + evidence_items: evidence_items, + contribution_form: + to_form(%{ + "rationale" => rationale, + "locator" => locator, + "claimant" => "me", + "language_tag" => "", + "valid_from" => "", + "valid_to" => "" + }) + ) + |> stream(:selected_evidence, evidence_rows, reset: true)} end defp preselected_subject(nil), do: nil @@ -186,6 +221,33 @@ defmodule DevilsDictionaryWeb.ConnectionLive do end end + defp preselected_evidence(nil, _locator), do: {%{}, []} + + defp preselected_evidence(value, locator) do + with {id, ""} when id > 0 <- Integer.parse(value), + true <- locator != "", + true <- + Repo.exists?( + from revision in DevilsDictionary.Corpus.SourceRecordRevision, + where: revision.id == ^id + ) do + item = %{ + id: "selected-evidence-source-#{id}", + object_id: nil, + label: "Artsy source observation", + detail: "Direct provider assignment retained for review", + target: %{source_record_revision_id: id}, + evidence_role: "supports", + locator: locator, + attribution_text: "Artsy direct artwork gene assignment" + } + + {%{item.id => item}, [item]} + else + _ -> {%{}, []} + end + end + defp revision_number(nil), do: {:ok, nil} defp revision_number(value) do diff --git a/lib/devils_dictionary_web/live/entity_live.ex b/lib/devils_dictionary_web/live/entity_live.ex index 18d3efae..2eae8084 100644 --- a/lib/devils_dictionary_web/live/entity_live.ex +++ b/lib/devils_dictionary_web/live/entity_live.ex @@ -21,12 +21,22 @@ defmodule DevilsDictionaryWeb.EntityLive do use DevilsDictionaryWeb, :live_view alias DevilsDictionary.Claims.Connection + alias DevilsDictionary.Artworks alias DevilsDictionary.Encyclopedia.EntityPage alias DevilsDictionary.Markdown @impl true def mount(_params, _session, socket), - do: {:ok, assign(socket, page: nil, id: nil, cursors: %{}, entity_slug: nil, back_path: nil)} + do: + {:ok, + assign(socket, + page: nil, + artwork: nil, + id: nil, + cursors: %{}, + entity_slug: nil, + back_path: nil + )} @impl true def handle_params(%{"id" => id, "slug" => slug} = params, _uri, socket) do @@ -51,6 +61,7 @@ defmodule DevilsDictionaryWeb.EntityLive do {:noreply, socket |> assign(:page, page) + |> assign(:artwork, Artworks.get(object_id)) |> assign(:id, object_id) |> assign(:cursors, cursors) |> assign(:entity_slug, canonical) @@ -69,6 +80,7 @@ defmodule DevilsDictionaryWeb.EntityLive do defp missing(socket, id) do socket |> assign(:page, nil) + |> assign(:artwork, nil) |> assign(:id, id) |> assign(:page_title, "no such thing") end @@ -138,23 +150,26 @@ defmodule DevilsDictionaryWeb.EntityLive do class="grid items-start gap-6 sm:grid-cols-[8rem_minmax(0,1fr)] sm:gap-8" >
{"Poster
- <.icon name="hero-film" class="size-7 stroke-current" /> + <.icon + name={if(@artwork, do: "hero-photo", else: "hero-film")} + class="size-7 stroke-current" + />
@@ -172,6 +187,54 @@ defmodule DevilsDictionaryWeb.EntityLive do
+
+ <.eyebrow>artwork record +
+
+
Creator
+
+ + 0}>, <.a navigate={ + ~p"/entities/#{creator.object_id}/#{Connection.slugify(creator.label)}" + }> + {creator.label} + + +
+
+
+
Date
{@artwork.date}
+
+
+
Medium
{@artwork.medium}
+
+
+
Collection
{@artwork.collection}
+
+
+
Image credit or rights notice
+
{@artwork.image_attribution}
+
+
+ +

+ Artsy fields are a freshness-limited source cache and may become stale. Dictionary keeps + Wikidata and independently supported identity data when that provider is unavailable. +

+
+ <.panel :if={@page.meaning_connections != []} id="entity-meaning-connections" diff --git a/lib/mix/tasks/dd.artworks.seed.ex b/lib/mix/tasks/dd.artworks.seed.ex index aae8757d..8c516f78 100644 --- a/lib/mix/tasks/dd.artworks.seed.ex +++ b/lib/mix/tasks/dd.artworks.seed.ex @@ -7,16 +7,26 @@ defmodule Mix.Tasks.Dd.Artworks.Seed do identity resolver. mix dd.artworks.seed --manifest priv/artworks/manifests/pilot-v1.json --build --dry-run - mix dd.artworks.seed --manifest priv/artworks/manifests/pilot-v1.json --resume + mix dd.artworks.seed --manifest priv/artworks/manifests/pilot-v1.json \ + --record-limit 50 --request-limit 160 \ + --wikidata-entity-limit 100 --wikidata-request-limit 4 --resume `--dry-run` never writes the database. `--build` discovers candidates and writes only the manifest; add `--import` to build and import in one command. - All discovery and provider calls are bounded. Reruns converge on exact - Wikidata, P11005/P2042 and opaque Artsy identifiers; titles never identify. + Imports hydrate selected artwork and creator QIDs through the existing bounded + Wikidata adapter before optional Artsy enrichment. Use + `--skip-wikidata-hydration` only when that stage is already complete, and + `--refresh-wikidata` when current Wikidata records must be refreshed. + + All discovery and provider calls are bounded. The manifest checkpoints every + collection page and identity/creator stage, so `--resume` continues partial + records without replaying completed pages. Reruns converge on exact Wikidata, + P11005/P2042 and opaque Artsy identifiers; titles never identify. """ use Mix.Task + alias DevilsDictionary.Artworks alias DevilsDictionary.Artworks.{Manifest, Seeder, WikidataCandidates} @switches [ @@ -31,7 +41,11 @@ defmodule Mix.Tasks.Dd.Artworks.Seed do discovery_request_limit: :integer, record_limit: :integer, batch_size: :integer, - offset: :integer + offset: :integer, + skip_wikidata_hydration: :boolean, + refresh_wikidata: :boolean, + wikidata_request_limit: :integer, + wikidata_entity_limit: :integer ] @impl Mix.Task @@ -49,6 +63,13 @@ defmodule Mix.Tasks.Dd.Artworks.Seed do if opts[:build] == true and opts[:import] != true do print_manifest(manifest, path) else + wikidata_stats = + if opts[:dry_run] || opts[:skip_wikidata_hydration] do + %{skipped: true, reason: if(opts[:dry_run], do: "dry_run", else: "operator_requested")} + else + hydrate_wikidata(manifest, opts) + end + {manifest, summary} = Seeder.run(manifest, dry_run: opts[:dry_run] || false, @@ -59,8 +80,13 @@ defmodule Mix.Tasks.Dd.Artworks.Seed do batch_size: opts[:batch_size] || 25 ) + mapping_stats = + if opts[:dry_run], do: %{skipped: "dry_run"}, else: Artworks.install_meaning_mappings!() + print_manifest(manifest, path) + Mix.shell().info("Wikidata hydration summary: " <> inspect(wikidata_stats, pretty: true)) Mix.shell().info("Artwork seed summary: " <> inspect(summary, pretty: true)) + Mix.shell().info("Meaning mapping summary: " <> inspect(mapping_stats, pretty: true)) end rescue error in [ArgumentError, File.Error] -> Mix.raise(Exception.message(error)) @@ -90,4 +116,28 @@ defmodule Mix.Tasks.Dd.Artworks.Seed do "Artwork manifest #{path}: #{length(manifest["candidates"])} candidates, checksum #{manifest["checksum"]}" ) end + + defp hydrate_wikidata(manifest, opts) do + qids = + manifest["candidates"] + |> Enum.flat_map(fn candidate -> + [candidate["qid"] | Enum.map(candidate["creators"] || [], & &1["qid"])] + end) + |> Enum.filter(&is_binary/1) + |> Enum.uniq() + + entity_limit = opts[:wikidata_entity_limit] || max(length(qids), 1) + request_limit = opts[:wikidata_request_limit] || max(div(length(qids) + 49, 50), 1) + + {:ok, stats} = + DevilsDictionary.Absorb.Sources.Wikidata.absorb(nil, + qids: Enum.take(qids, entity_limit), + entity_budget: entity_limit, + request_budget: request_limit, + related_depth: 1, + refresh: opts[:refresh_wikidata] || false + ) + + stats + end end diff --git a/priv/artworks/manifests/repair-new-entry-v3.json b/priv/artworks/manifests/repair-new-entry-v3.json new file mode 100644 index 00000000..6942e081 --- /dev/null +++ b/priv/artworks/manifests/repair-new-entry-v3.json @@ -0,0 +1,65 @@ +{ + "candidates": [ + { + "artsy_artwork_slug": "ferdinand-hodler-the-disappointed-souls-les-ames-decues", + "creators": [ + { + "artsy_artist_slug": "ferdinand-hodler", + "name": "Ferdinand Hodler", + "qid": "Q214564" + } + ], + "description": "1892 painting by Ferdinand Hodler", + "import": { + "object_id": 3733934, + "reason": "artsy_artwork_not_found", + "stages": { + "artists": { + "next_cursor": null, + "status": "pending" + }, + "artwork": { + "status": "unavailable" + }, + "creators": { + "status": "pending" + }, + "genes": { + "next_cursor": null, + "status": "pending" + }, + "identity": { + "status": "pending" + }, + "wikidata": { + "preserved": true, + "source_record_id": 6984052, + "source_record_revision_id": 6984052, + "status": "materialized_full_record" + } + }, + "status": "unavailable", + "updated_at": "2026-09-12T19:28:24Z" + }, + "kind": "painting", + "qid": "Q122978100", + "selection_reason": "wikidata_p11005_painting", + "title": "Les Âmes Déçues", + "wikipedia_title": null + } + ], + "checksum": "206b3b4691c739f14a9e68416cf57e5aac484db3251fcde4c410af7595e9f381", + "generated_at": "2026-09-12T19:28:05Z", + "schema_version": 2, + "selection": { + "artwork_kinds": [ + "painting" + ], + "identity": "Wikidata QID plus P11005; title is never identity", + "pilot_cap": 1, + "selection_reason": "exact Wikidata P11005 painting; bounded independent audit repair", + "source": "issue-86 repair new-entry proof", + "wikipedia_required": false + }, + "updated_at": "2026-09-12T19:35:20Z" +} diff --git a/priv/artworks/manifests/repair-pilot-v2.json b/priv/artworks/manifests/repair-pilot-v2.json new file mode 100644 index 00000000..d5601400 --- /dev/null +++ b/priv/artworks/manifests/repair-pilot-v2.json @@ -0,0 +1,140 @@ +{ + "candidates": [ + { + "artsy_artwork_slug": "titian-cupid-with-the-wheel-of-fortune", + "creators": [ + { + "artsy_artist_slug": "titian", + "name": "Titian", + "qid": "Q47551" + } + ], + "description": "painting by Titian", + "import": { + "artsy_artwork_id": "515d39335eeb1c524c0039df", + "artsy_artwork_slug": "titian-cupid-with-the-wheel-of-fortune", + "cache_fresh": true, + "creator_count": 1, + "creator_links": 0, + "gene_count": 13, + "missing_creators": 0, + "object_id": 3733930, + "source_record_id": 6984025, + "stages": { + "artists": { + "next_cursor": null, + "pages": 1, + "returned_next_preserved_filter": true, + "status": "complete" + }, + "artwork": { + "status": "complete" + }, + "creators": { + "missing": 0, + "resolved": 1, + "status": "complete" + }, + "genes": { + "next_cursor": null, + "pages": 1, + "returned_next_preserved_filter": true, + "status": "complete" + }, + "identity": { + "status": "complete" + }, + "wikidata": { + "preserved": true, + "source_record_id": 6984024, + "source_record_revision_id": 6984036, + "status": "materialized_full_record" + } + }, + "status": "matched", + "updated_at": "2026-09-12T19:32:52Z" + }, + "kind": "painting", + "origin": "local_registry", + "qid": "Q12115903", + "selection_reason": "existing_local_wikidata_p11005", + "title": "Cupid with the Wheel of Time", + "wikipedia_title": null + }, + { + "artsy_artwork_slug": "titian-ranuccio-farnese-1", + "creators": [ + { + "artsy_artist_slug": "titian", + "name": "Titian", + "qid": "Q47551" + } + ], + "description": "painting by Titian", + "import": { + "artsy_artwork_id": "515b4aee38ad2d25a7001435", + "artsy_artwork_slug": "titian-ranuccio-farnese-1", + "cache_fresh": true, + "creator_count": 1, + "creator_links": 0, + "gene_count": 15, + "missing_creators": 0, + "object_id": 3733932, + "source_record_id": 6984048, + "stages": { + "artists": { + "next_cursor": null, + "pages": 1, + "returned_next_preserved_filter": true, + "status": "complete" + }, + "artwork": { + "status": "complete" + }, + "creators": { + "missing": 0, + "resolved": 1, + "status": "complete" + }, + "genes": { + "next_cursor": null, + "pages": 1, + "returned_next_preserved_filter": true, + "status": "complete" + }, + "identity": { + "status": "complete" + }, + "wikidata": { + "preserved": true, + "source_record_id": 6984038, + "source_record_revision_id": 6984038, + "status": "materialized_full_record" + } + }, + "status": "matched", + "updated_at": "2026-09-12T19:32:52Z" + }, + "kind": "painting", + "origin": "wikidata_sparql", + "qid": "Q12146958", + "selection_reason": "wikidata_p11005_with_english_wikipedia", + "title": "Portrait of Ranuccio Farnese", + "wikipedia_title": "Portrait of Ranuccio Farnese" + } + ], + "checksum": "a86427de706045c3fb9fd3c6defd8e9503ace24dbf7c6722757a731b7fe6fca7", + "generated_at": "2026-09-12T21:30:00Z", + "schema_version": 2, + "selection": { + "artwork_kinds": [ + "painting" + ], + "identity": "Wikidata QID plus P11005; title is never identity", + "pilot_cap": 2, + "selection_reason": "bounded repair verification: one existing local identity and one missing local candidate", + "source": "Wikidata P11005/P2042", + "wikipedia_required": false + }, + "updated_at": "2026-09-12T19:32:52Z" +} diff --git a/priv/artworks/meaning-mappings-v1.json b/priv/artworks/meaning-mappings-v1.json index 1489faea..31fbc1d0 100644 --- a/priv/artworks/meaning-mappings-v1.json +++ b/priv/artworks/meaning-mappings-v1.json @@ -1,96 +1,136 @@ { - "version": 1, - "policy": "Exact sense and language mappings only. Artsy genes are attributed source vocabulary; these rows create candidates, never accepted illustrates claims.", + "version": 2, + "policy": "Mappings target one stable source sense and one verified opaque Artsy gene ID. They create review candidates only; names and glosses are display text, never identity.", "mappings": [ { - "version": 1, + "version": 2, + "enabled": true, + "sense_source_slug": "wordnet", + "sense_external_key": "oewn-00975181-n#war", "lemma": "war", "language": "en", - "gloss_contains": ["armed conflict", "warfare", "hostilities"], + "gene_id": "4db9b645db68d133c600123e", "gene_name": "Conflict", "match_type": "related", - "note": "A direct Conflict assignment is related evidence, not proof that the work illustrates this exact sense." + "mapping_review_status": "approved_for_candidate_generation_only", + "mapping_review_actor": "issue-86 bounded feasibility review", + "mapping_reviewed_at": "2026-09-12", + "mapping_review_basis": "Exact Artsy gene resource and exact Open English WordNet source sense were verified; semantic use remains related and requires per-artwork review.", + "note": "Conflict is related subject vocabulary. It does not establish that a work illustrates armed warfare; a reviewer must inspect the depicted event." }, { - "version": 1, + "version": 2, + "enabled": true, + "sense_source_slug": "wordnet", + "sense_external_key": "oewn-07986853-n#family", "lemma": "family", "language": "en", - "gloss_contains": ["related", "parents", "children", "household"], + "gene_id": "50b99005c2f9a4a28b00015a", "gene_name": "Family", - "match_type": "exact", - "note": "The exact sense still requires review of the depicted relationship." + "match_type": "related", + "mapping_review_status": "approved_for_candidate_generation_only", + "mapping_review_actor": "issue-86 bounded feasibility review", + "mapping_reviewed_at": "2026-09-12", + "mapping_review_basis": "Exact Artsy gene resource and exact Open English WordNet source sense were verified; semantic use remains related and requires per-artwork review.", + "note": "Family is related subject vocabulary; review must verify the depicted relationship for this specific social-group sense." + }, + { + "version": 2, + "enabled": true, + "sense_source_slug": "wordnet", + "sense_external_key": "oewn-01157387-n#nepotism", + "lemma": "nepotism", + "language": "en", + "gene_id": "50b99005c2f9a4a28b00015a", + "gene_name": "Family", + "match_type": "broader", + "mapping_review_status": "approved_for_candidate_generation_only", + "mapping_review_actor": "issue-86 bounded feasibility review", + "mapping_reviewed_at": "2026-09-12", + "mapping_review_basis": "Exact IDs were verified and the Family-to-nepotism gap was explicitly classified as broader exploratory context, never direct evidence.", + "note": "Family is only broader exploratory context. It is not evidence of favoritism, appointment, or nepotism without independent review evidence." }, { - "version": 1, + "version": 2, + "enabled": true, + "sense_source_slug": "wordnet", + "sense_external_key": "oewn-07558676-n#love", "lemma": "love", "language": "en", - "gloss_contains": ["affection", "romantic", "attachment"], + "gene_id": "4de292fcef72520001005fe5", "gene_name": "Love", - "match_type": "exact", - "note": "Provider vocabulary is candidate evidence only." + "match_type": "related", + "mapping_review_status": "approved_for_candidate_generation_only", + "mapping_review_actor": "issue-86 bounded feasibility review", + "mapping_reviewed_at": "2026-09-12", + "mapping_review_basis": "Exact Artsy gene resource and exact Open English WordNet source sense were verified; semantic use remains related and requires per-artwork review.", + "note": "The provider subject is related evidence only; a reviewer must distinguish affection, romance, attachment, and other senses." }, { - "version": 1, + "version": 2, + "enabled": false, + "sense_source_slug": "wordnet", + "sense_external_key": "oewn-07550398-n#grief", "lemma": "grief", "language": "en", - "gloss_contains": ["sorrow", "loss", "sadness"], - "gene_name": "Grief", - "match_type": "exact", - "note": "Provider vocabulary is candidate evidence only." + "gene_slug_probe": "grief", + "probe_result": "404 on 2026-09-12", + "note": "Disabled: no stable Artsy gene ID was verified within the bounded feasibility audit." }, { - "version": 1, + "version": 2, + "enabled": false, + "sense_source_slug": "wordnet", + "sense_external_key": "oewn-14438384-n#solitude", "lemma": "solitude", "language": "en", - "gloss_contains": ["alone", "seclusion", "solitary"], - "gene_name": "Solitude", - "match_type": "exact", - "note": "A title match alone never satisfies this mapping." + "gene_slug_probe": "solitude", + "probe_result": "404 on 2026-09-12", + "note": "Disabled: no stable Artsy gene ID was verified within the bounded feasibility audit." }, { - "version": 1, + "version": 2, + "enabled": false, + "sense_source_slug": "wordnet", + "sense_external_key": "oewn-04857348-n#justice", "lemma": "justice", "language": "en", - "gloss_contains": ["fair", "law", "right"], - "gene_name": "Justice", - "match_type": "exact", - "note": "The allegorical or political sense must be reviewed." + "gene_slug_probe": "justice", + "probe_result": "404 on 2026-09-12", + "note": "Disabled: no stable Artsy gene ID was verified within the bounded feasibility audit." }, { - "version": 1, + "version": 2, + "enabled": false, + "sense_source_slug": "wordnet", + "sense_external_key": "oewn-05198072-n#power", "lemma": "power", "language": "en", - "gloss_contains": ["control", "authority", "influence"], - "gene_name": "Power", - "match_type": "exact", - "note": "Wrong-sense physical-power matches must be dismissed." - }, - { - "version": 1, - "lemma": "nepotism", - "language": "en", - "gloss_contains": ["relative", "family", "favoritism", "favouritism"], - "gene_name": "Family", - "match_type": "broader", - "note": "Family is exploratory context, not evidence of favoritism or nepotism." + "gene_slug_probe": "power", + "probe_result": "404 on 2026-09-12", + "note": "Disabled: no stable Artsy gene ID was verified within the bounded feasibility audit." }, { - "version": 1, + "version": 2, + "enabled": false, + "sense_source_slug": "wordnet", + "sense_external_key": "oewn-11464213-n#death", "lemma": "death", "language": "en", - "gloss_contains": ["end of life", "dying", "ceasing to live"], - "gene_name": "Death", - "match_type": "exact", - "note": "A direct Death assignment proposes a candidate; the depicted event and exact lexical sense still require review." + "gene_slug_probe": "death", + "probe_result": "404 on 2026-09-12", + "note": "Disabled: no stable Artsy gene ID was verified within the bounded feasibility audit." }, { - "version": 1, + "version": 2, + "enabled": false, + "sense_source_slug": "wordnet", + "sense_external_key": "oewn-14015308-n#freedom", "lemma": "freedom", "language": "en", - "gloss_contains": ["liberty", "free", "restraint"], - "gene_name": "Freedom", - "match_type": "exact", - "note": "Political, physical, and allegorical freedom must not be conflated during review." + "gene_slug_probe": "freedom", + "probe_result": "404 on 2026-09-12", + "note": "Disabled: no stable Artsy gene ID was verified within the bounded feasibility audit." } ] } diff --git a/test/devils_dictionary/absorb/sources/wikidata_test.exs b/test/devils_dictionary/absorb/sources/wikidata_test.exs index e23b885b..ab0e9925 100644 --- a/test/devils_dictionary/absorb/sources/wikidata_test.exs +++ b/test/devils_dictionary/absorb/sources/wikidata_test.exs @@ -108,7 +108,8 @@ defmodule DevilsDictionary.Absorb.Sources.WikidataTest do } } - assert [work] = out(artwork).concepts + artwork_output = out(artwork) + assert [work] = artwork_output.concepts assert work.kind == :work assert work.work_kind == "artwork" assert work.metadata["wikidata_creators"] == ["Q900087"] @@ -118,6 +119,11 @@ defmodule DevilsDictionary.Absorb.Sources.WikidataTest do &(&1.namespace == "artsy_artwork_slug" and &1.external_id == "fixture-work") ) + assert Enum.any?(artwork_output.concept_relations, fn relation -> + relation.type == :authored_by and relation.from_concept == "Q900086" and + relation.to_concept == "Q900087" + end) + assert [person] = out(artist).concepts assert person.kind == :person diff --git a/test/devils_dictionary/artsy/client_test.exs b/test/devils_dictionary/artsy/client_test.exs index 4811c87d..ac1257d9 100644 --- a/test/devils_dictionary/artsy/client_test.exs +++ b/test/devils_dictionary/artsy/client_test.exs @@ -1,7 +1,7 @@ defmodule DevilsDictionary.Artsy.ClientTest do use ExUnit.Case, async: true - alias DevilsDictionary.Artsy.Client + alias DevilsDictionary.Artsy.{Client, RequestCoordinator} test "reports stale search hits and rebuilds pagination with the artwork filter" do responses = [ @@ -95,6 +95,48 @@ defmodule DevilsDictionary.Artsy.ClientTest do refute normalized["id"] == normalized["slug"] end + test "Retry-After is retained in full and shared with every client" do + clock = start_supervised!({Agent, fn -> 0 end}, id: make_ref()) + + coordinator = + start_supervised!( + {RequestCoordinator, + name: nil, interval_ms: 0, now_fun: fn -> Agent.get(clock, & &1) end}, + id: make_ref() + ) + + response = response(429, %{}) |> Req.Response.put_header("retry-after", "120") + + {client, _calls} = + client([response(201, %{"token" => "fixture-token"}), response], + coordinator: coordinator + ) + + assert {:error, %{code: "quota_exhausted"}, _client} = Client.artwork(client, "work-one") + assert RequestCoordinator.stats(coordinator).next_at == 120_000 + end + + test "shared coordinator serializes attempts across otherwise independent clients" do + # System.monotonic_time/1 may be negative; initialization must use the same + # clock rather than assuming a zero epoch and sleeping for years. + clock = start_supervised!({Agent, fn -> -1_000 end}, id: make_ref()) + + coordinator = + start_supervised!( + {RequestCoordinator, + name: nil, interval_ms: 250, now_fun: fn -> Agent.get(clock, & &1) end}, + id: make_ref() + ) + + assert {:ok, generation, 0} = RequestCoordinator.acquire(coordinator, 250) + assert {:ok, ^generation, 250} = RequestCoordinator.acquire(coordinator, 250) + assert RequestCoordinator.stats(coordinator).attempts == 2 + + assert {:ok, _new_generation} = RequestCoordinator.disable(coordinator) + refute RequestCoordinator.current?(generation, coordinator) + assert {:error, :provider_disabled} = RequestCoordinator.acquire(coordinator, 250) + end + defp client(responses, opts \\ []) do calls = start_supervised!({Agent, fn -> [] end}) queue = start_supervised!({Agent, fn -> responses end}, id: make_ref()) diff --git a/test/devils_dictionary/artworks/seeder_test.exs b/test/devils_dictionary/artworks/seeder_test.exs index 3151a8d0..1836652e 100644 --- a/test/devils_dictionary/artworks/seeder_test.exs +++ b/test/devils_dictionary/artworks/seeder_test.exs @@ -7,6 +7,7 @@ defmodule DevilsDictionary.Artworks.SeederTest do alias DevilsDictionary.Claims.Assertion alias DevilsDictionary.Registry alias DevilsDictionary.Registry.{Entity, ExternalIdentifier, WorkDetails} + alias DevilsDictionary.Sources alias DevilsDictionary.Sources.SourceRecord alias DevilsDictionary.Sources.ReconciliationCase @@ -102,6 +103,161 @@ defmodule DevilsDictionary.Artworks.SeederTest do assert summary.requests == 0 end + test "request exhaustion checkpoints artwork and resumes the same manifest without duplicates" do + {partial_manifest, partial_summary} = + Seeder.run(Manifest.new([candidate()]), + artsy_client: + client( + [token(), response(200, artwork("opaque-work-86", "fixture-work", "Painting"))], + request_limit: 2 + ), + record_limit: 1 + ) + + assert partial_summary.request_exhausted + assert partial_summary.partial == 1 + refute Manifest.completed?(hd(partial_manifest["candidates"])) + + assert get_in(partial_manifest, [ + "candidates", + Access.at(0), + "import", + "stages", + "artists", + "status" + ]) == "partial" + + {completed_manifest, resumed} = + Seeder.run(partial_manifest, + artsy_client: + client([ + token(), + Enum.at(painting_responses(), 2), + Enum.at(painting_responses(), 3) + ]), + record_limit: 1, + resume: true + ) + + assert resumed.matched == 1 + assert Manifest.completed?(hd(completed_manifest["candidates"])) + + {_same_manifest, rerun} = + Seeder.run(completed_manifest, + artsy_client: client([]), + record_limit: 1, + resume: true + ) + + assert rerun.requests == 0 + assert rerun.processed == 0 + + assert Repo.aggregate( + from(identifier in ExternalIdentifier, + where: identifier.namespace == "artsy_artwork_id" + ), + :count + ) == 1 + end + + test "seeding materializes but never replaces a rich current Wikidata record" do + %{sources: sources} = Sources.Catalog.seed!() + + rich = %{ + "id" => "Q900086", + "labels" => %{"en" => %{"language" => "en", "value" => "Fixture Work"}}, + "descriptions" => %{"en" => %{"language" => "en", "value" => "Source description"}}, + "claims" => %{ + "P31" => [entity_statement("Q3305213")], + "P18" => [string_statement("Picture.jpg")], + "P11005" => [string_statement("fixture-work")] + } + } + + {:ok, record} = + Sources.upsert_record(sources["wikidata"], %{external_id: "Q900086", raw: rich}) + + original_hash = record.content_hash + + {_manifest, _summary} = + Seeder.run(Manifest.new([candidate()]), artsy_client: client(painting_responses())) + + record = Repo.get!(SourceRecord, record.id) + assert record.content_hash == original_hash + assert Sources.raw(record) == rich + end + + test "Wikidata creator link remains when the exact Artsy artwork endpoint is missing" do + {_manifest, summary} = + Seeder.run(Manifest.new([candidate()]), + artsy_client: client(unavailable_responses()), + record_limit: 1 + ) + + assert summary.unavailable == 1 + work_id = Registry.by_external_id("wikidata", "Q900086") + creator_id = Registry.by_external_id("wikidata", "Q900087") + assert Enum.any?(Artworks.get(work_id).creators, &(&1.object_id == creator_id)) + end + + test "artist collection pagination retains multiple exact creators" do + candidate = + put_in(candidate()["creators"], [ + %{ + "qid" => "Q900087", + "name" => "Fixture Artist", + "artsy_artist_slug" => "fixture-artist" + }, + %{ + "qid" => "Q900088", + "name" => "Second Artist", + "artsy_artist_slug" => "second-artist" + } + ]) + + first_page = + response(200, %{ + "_embedded" => %{ + "artists" => [artist("opaque-artist-87", "fixture-artist", "Fixture Artist")] + }, + "_links" => %{ + "next" => %{ + "href" => + "https://api.artsy.net/api/artists?artwork_id=opaque-work-86&size=50&offset=1" + } + } + }) + + second_page = + response(200, %{ + "_embedded" => %{ + "artists" => [artist("opaque-artist-88", "second-artist", "Second Artist")] + } + }) + + genes = response(200, %{"_embedded" => %{"genes" => []}}) + + {_manifest, summary} = + Seeder.run(Manifest.new([candidate]), + artsy_client: + client([ + token(), + response(200, artwork("opaque-work-86", "fixture-work", "Painting")), + first_page, + second_page, + genes + ]) + ) + + assert summary.creators_resolved == 2 + work_id = Registry.by_external_id("wikidata", "Q900086") + + assert Enum.map(Artworks.get(work_id).creators, & &1.label) == [ + "Fixture Artist", + "Second Artist" + ] + end + test "withdrawal succeeds after a seeded creator citation and keeps durable QIDs" do {_manifest, _summary} = Seeder.run(Manifest.new([candidate()]), @@ -115,14 +271,31 @@ defmodule DevilsDictionary.Artworks.SeederTest do assert {:ok, summary} = Artworks.withdraw_artsy("fixture provider shutdown") assert summary.claims_withdrawn == 1 assert summary.evidence_removed == 1 - assert summary.payloads_deleted == 2 + # Artwork collection checkpoints are immutable revisions too; withdrawal + # removes the entire provider-owned history, not only its final snapshot. + assert summary.payloads_deleted >= 2 assert Registry.by_external_id("wikidata", "Q900086") == work_id assert Registry.by_external_id("wikidata", "Q900087") == artist_id + assert Registry.by_external_id("artsy_artwork_slug", "fixture-work") == work_id + assert Registry.by_external_id("artsy_artist_slug", "fixture-artist") == artist_id assert Registry.by_external_id("artsy_artwork_id", "opaque-work-86") == nil assert Registry.by_external_id("artsy_artist_id", "opaque-artist-87") == nil + + record_count = Repo.aggregate(SourceRecord, :count) + + {_manifest, stopped} = + Seeder.run(Manifest.new([candidate()]), + artsy_client: client(painting_responses()), + record_limit: 1 + ) + + assert stopped.provider_disabled + assert stopped.requests == 0 + assert stopped.processed == 0 + assert Repo.aggregate(SourceRecord, :count) == record_count end - defp client(responses) do + defp client(responses, opts \\ []) do queue = start_supervised!({Agent, fn -> responses end}, id: make_ref()) request_fun = fn _options -> @@ -133,12 +306,19 @@ defmodule DevilsDictionary.Artworks.SeederTest do end Client.new( - client_id: "fixture-id", - client_secret: "fixture-secret", - request_fun: request_fun, - sleep_fun: fn _ -> :ok end, - rate_limit_ms: 0, - request_limit: 20 + Keyword.merge( + [ + client_id: "fixture-id", + client_secret: "fixture-secret", + request_fun: request_fun, + sleep_fun: fn _ -> :ok end, + rate_limit_ms: 0, + request_limit: 20, + availability_fun: fn -> :ok end, + coordinator: nil + ], + opts + ) ) end @@ -202,6 +382,35 @@ defmodule DevilsDictionary.Artworks.SeederTest do } end + defp artist(id, slug, name) do + %{ + "id" => id, + "slug" => slug, + "name" => name, + "_links" => %{ + "permalink" => %{"href" => "https://www.artsy.net/artist/#{slug}"} + } + } + end + defp token, do: response(201, %{"token" => "fixture-token"}) defp response(status, body), do: %Req.Response{status: status, body: body} + + defp entity_statement(qid) do + %{ + "type" => "statement", + "rank" => "normal", + "mainsnak" => %{ + "datavalue" => %{"value" => %{"id" => qid}, "type" => "wikibase-entityid"} + } + } + end + + defp string_statement(value) do + %{ + "type" => "statement", + "rank" => "normal", + "mainsnak" => %{"datavalue" => %{"value" => value, "type" => "string"}} + } + end end diff --git a/test/devils_dictionary_web/live/artwork_live_test.exs b/test/devils_dictionary_web/live/artwork_live_test.exs index 5e93fd48..32beae16 100644 --- a/test/devils_dictionary_web/live/artwork_live_test.exs +++ b/test/devils_dictionary_web/live/artwork_live_test.exs @@ -32,9 +32,16 @@ defmodule DevilsDictionaryWeb.ArtworkLiveTest do "id" => "opaque-live-86", "slug" => "fixture-of-war", "title" => "The Fixture of War", + "date" => "1814", + "medium" => "Oil on canvas", + "collecting_institution" => "Fixture Museum", + "thumbnail_url" => "https://images.example.test/fixture.jpg", + "image_rights" => "Fixture credit", "permalink" => "https://www.artsy.net/artwork/fixture-of-war" }, - "genes" => [%{"id" => "gene-conflict", "name" => "Conflict"}] + "genes" => [ + %{"id" => "4db9b645db68d133c600123e", "name" => "Conflict"} + ] } }) @@ -81,8 +88,43 @@ defmodule DevilsDictionaryWeb.ArtworkLiveTest do html = view |> form("#artwork-search", search: %{q: "Fixture Painter"}) |> render_change() assert html =~ ~s(id="artwork-#{ctx.work.object_id}") + html = view |> form("#artwork-search", search: %{q: "Conflict"}) |> render_change() + assert html =~ ~s(id="artwork-#{ctx.work.object_id}") + html = view |> form("#artwork-search", search: %{q: "no such work"}) |> render_change() refute html =~ ~s(id="artwork-#{ctx.work.object_id}") + + {:ok, unrelated} = + Registry.create_work(%{ + preferred_label: "Unrelated Search Target", + work_kind: "book" + }) + + {:ok, _claim} = Claims.assert(ctx.work.object_id, "adaptation_of", unrelated.object_id) + + html = + view + |> form("#artwork-search", search: %{q: "Unrelated Search Target"}) + |> render_change() + + refute html =~ ~s(id="artwork-#{ctx.work.object_id}") + end + + test "catalog pages beyond the first 24 reusable artworks", ctx do + for index <- 1..24 do + {:ok, _work} = + Registry.create_work(%{ + preferred_label: "Page Fixture #{String.pad_leading(to_string(index), 2, "0")}", + work_kind: "artwork" + }) + end + + {:ok, view, _html} = live(ctx.conn, ~p"/artworks") + assert has_element?(view, "#artwork-pagination") + refute has_element?(view, "#artwork-#{ctx.work.object_id}") + + view |> element("#artwork-next-page") |> render_click() + assert has_element?(view, "#artwork-#{ctx.work.object_id}") end test "a contributor opens the existing composer with the exact artwork selected", ctx do @@ -97,7 +139,14 @@ defmodule DevilsDictionaryWeb.ArtworkLiveTest do test "a direct gene produces a labeled candidate for the exact sense, never a claim", ctx do war = word!(ctx, "war", ["wordnet"]) - sense = sense!(ctx, war, "wordnet", gloss: "organized armed conflict or warfare") + + sense = + sense!(ctx, war, "wordnet", + external_id: "oewn-00975181-n#war", + gloss: "organized armed conflict or warfare" + ) + + assert %{installed: 1} = Artworks.install_meaning_mappings!() {:ok, view, _html} = live(ctx.conn, ~p"/define/war") assert has_element?(view, "#artwork-candidates") @@ -108,6 +157,20 @@ defmodule DevilsDictionaryWeb.ArtworkLiveTest do ) assert render(view) =~ "not yet reviewed" + + wrong_id_payload = + Map.put(ctx.record.raw, "genes", [ + %{"id" => "not-the-conflict-gene", "name" => "Conflict"} + ]) + + {:ok, _record} = + Sources.upsert_record(ctx.sources["artsy"], %{ + external_id: ctx.record.external_id, + url: ctx.record.url, + raw: wrong_id_payload + }) + + assert Artworks.suggestions([war.object_id]) == [] end test "provider withdrawal deletes Artsy payloads but preserves Wikidata identity", ctx do @@ -129,4 +192,52 @@ defmodule DevilsDictionaryWeb.ArtworkLiveTest do assert Artworks.search("Fixture") |> Enum.any?(&(&1.object_id == ctx.work.object_id)) assert Artworks.get(ctx.work.object_id).artsy == nil end + + test "artwork page exposes rich metadata and a working creator destination", ctx do + {:ok, view, _html} = + live( + ctx.conn, + ~p"/entities/#{ctx.work.object_id}/#{DevilsDictionary.Claims.Connection.slugify(ctx.work.preferred_label)}" + ) + + assert has_element?(view, "#artwork-metadata") + assert has_element?(view, "#artwork-metadata a[href^='/entities/#{ctx.creator.object_id}/']") + assert render(view) =~ "Oil on canvas" + assert render(view) =~ "Fixture Museum" + end + + test "candidate link preselects exact meaning, predicate, rationale and provider revision", + ctx do + war = word!(ctx, "war", ["wordnet"]) + + sense = + sense!(ctx, war, "wordnet", + external_id: "oewn-00975181-n#war", + gloss: "organized armed conflict or warfare" + ) + + %{conn: conn, user: user} = register_and_log_in_user(%{conn: ctx.conn}) + _user = Repo.update!(Ecto.Changeset.change(user, internal_contributor: true)) + + path = + ~p"/connect?#{%{subject: ctx.work.object_id, object: sense.object_id, predicate: "illustrates", rationale: "Related provider vocabulary; inspect the depicted event.", evidence_revision: ctx.record.current_revision.id, evidence_locator: "Artsy direct gene 4db9b645db68d133c600123e"}}" + + {:ok, view, _html} = live(conn, path) + assert has_element?(view, "#composer-subject-chosen") + assert has_element?(view, "#composer-object-chosen") + assert has_element?(view, "#predicate-illustrates") + assert has_element?(view, "#selected-evidence-source-#{ctx.record.current_revision.id}") + + assert render(view) =~ "Related provider vocabulary; inspect the depicted event." + + refute ctx.work.object_id == sense.object_id + end + + test "rejected creator relationship is hidden from cards and search", ctx do + revision = Claims.current_revision(ctx.claim.id) + {:ok, _review} = Claims.review(revision.id, :rejected) + + assert Artworks.get(ctx.work.object_id).creators == [] + refute Enum.any?(Artworks.search("Fixture Painter"), &(&1.object_id == ctx.work.object_id)) + end end From df31ef85add011dadc18e1ef528f6c2d51d24cc6 Mon Sep 17 00:00:00 2001 From: Holden Thomas Date: Sat, 12 Sep 2026 21:56:33 +0200 Subject: [PATCH 3/8] Address CodeRabbit artwork audit findings --- .gitignore | 3 + .../artsy-feasibility-2026-09-12.md | 15 +- lib/devils_dictionary/artsy/availability.ex | 6 +- lib/devils_dictionary/artsy/client.ex | 37 +++- .../artsy/request_coordinator.ex | 60 +++++-- lib/devils_dictionary/artworks.ex | 1 + lib/devils_dictionary/artworks/manifest.ex | 7 + lib/devils_dictionary/artworks/seeder.ex | 160 +++++++++++++----- .../artworks/wikidata_candidates.ex | 5 +- .../live/artwork_live.ex | 20 +-- .../live/connection_live.ex | 8 +- lib/mix/tasks/dd.artworks.seed.ex | 42 +++-- priv/artworks/manifests/pilot-v1.json | 42 +---- .../manifests/repair-new-entry-v3.json | 36 +--- priv/artworks/manifests/repair-pilot-v2.json | 92 +--------- test/devils_dictionary/artsy/client_test.exs | 96 ++++++++++- .../artworks/seeder_test.exs | 103 +++++++++++ .../artworks/wikidata_candidates_test.exs | 29 ++++ .../live/artwork_live_test.exs | 4 +- 19 files changed, 513 insertions(+), 253 deletions(-) create mode 100644 test/devils_dictionary/artworks/wikidata_candidates_test.exs diff --git a/.gitignore b/.gitignore index 29d0b4e7..57b4de53 100644 --- a/.gitignore +++ b/.gitignore @@ -45,6 +45,9 @@ npm-debug.log /priv/replay/* !/priv/replay/MANIFEST.json +# Artwork import progress is host-local; committed manifests remain portable inputs. +/priv/artworks/manifests/*.checkpoint.json + # Local development credentials. /.env /.env.* diff --git a/docs/integrations/artsy-feasibility-2026-09-12.md b/docs/integrations/artsy-feasibility-2026-09-12.md index a80d1ac4..9cd5e93b 100644 --- a/docs/integrations/artsy-feasibility-2026-09-12.md +++ b/docs/integrations/artsy-feasibility-2026-09-12.md @@ -13,6 +13,9 @@ identity. Independently licensed museum metadata and images, starting with the M are preferred for durable display. Artsy descriptions, gene assignments, image URLs, and cached responses must remain source-specific and removable. +This is a feasibility/pilot decision only. Merge, deployment, and issue closure remain +blocked until an independent audit verifies the implementation and its evidence. + The first pilot stays capped at 500 manifest candidates and 500 created/matched records, with smaller operator-configured request and batch limits. Given the live search hydration rate, start with Wikidata P11005-linked records rather than broad @@ -54,7 +57,7 @@ title search. Widen only after the pilot reports its actual coverage. | P2042 artist crosswalk | Wikidata P2042 is an artist slug | `vincent-van-gogh` redirected, then returned opaque ID `4d8b92944eb68a1b2c000264` | Resolve creators through exact slug/ID evidence, never through a name string. | | Direct gene assignments | Artwork links expose assigned genes | Goya returned 5 assignments: 19th Century, Chiaroscuro, Collective History, Conflict, Cultural Commentary | Preserve as attributed Artsy vocabulary. No string-to-meaning equivalence. | | Gene-filtered acquisition | Gene resources link to artworks | The first 10 `gene_id=19th Century` artwork IDs were identical to the unfiltered first 10 | **Unverified/unsafe.** Do not seed through gene traversal. A direct per-work assignment may label a review candidate only. | -| Retry/quota signal | 5 requests/sec documented | Probe throttled below 3/sec; 0 retries, 0 quota responses | One application-wide coordinator paces all callers. It honors the full `Retry-After`, bounds retries and total attempts, and reports local request exhaustion separately from provider quota exhaustion. | +| Retry/quota signal | 5 requests/sec documented | Probe throttled below 3/sec; 0 retries, 0 quota responses | One application-wide coordinator paces all callers. It honors the full `Retry-After`, bounds retries and total attempts, and reports local request exhaustion separately from provider quota exhaustion. Interactive discovery has one atomic 30-attempt server-boot allowance that browser reloads and concurrent LiveViews cannot reset. | ## Repair feasibility findings @@ -78,15 +81,17 @@ The final repair pilot used one bounded Wikidata request and seven Artsy attempt two manifest records. Both exact identities matched. One previously enriched record was reused and one existing Wikidata artwork was newly hydrated with Artsy metadata; no local identity was created, no conflict/reproduction/quota event occurred, and one -creator link was added. A resume of the saved manifest processed zero records, made -zero Artsy requests, and created no duplicates. +creator link was added. A resume of the environment-local checkpoint processed zero +records, made zero Artsy requests, and created no duplicates. The committed repair +manifest remains a portable, pending input rather than carrying database row IDs. A final one-candidate proof started from Q122978100, which was absent from the local catalog. One bounded Wikidata request materialized *Les Âmes Déçues* and its creator, Ferdinand Hodler, as reusable local identities with an `authored_by` route and source link. The exact Artsy endpoint was unavailable after three attempts, so no Artsy -fields or image were invented. Its saved manifest records `unavailable`; resuming it -made zero requests and no duplicates. This deliberately separates “new local identity +fields or image were invented. Its environment-local checkpoint recorded +`unavailable`; resuming it made zero requests and no duplicates. The committed +one-candidate manifest remains pending and portable. This deliberately separates “new local identity created from independent Wikidata evidence” from the Artsy import summary's honest `created: 0, unavailable: 1`. diff --git a/lib/devils_dictionary/artsy/availability.ex b/lib/devils_dictionary/artsy/availability.ex index aafd29dc..33368d7c 100644 --- a/lib/devils_dictionary/artsy/availability.ex +++ b/lib/devils_dictionary/artsy/availability.ex @@ -46,6 +46,7 @@ defmodule DevilsDictionary.Artsy.Availability do status_with_credentials(client_id, client_secret, DevilsDictionary.Artsy.RequestCoordinator) end + @doc "The explicit-credential lifecycle check with an injectable request coordinator." def status_with_credentials(client_id, client_secret, coordinator) do cond do config()[:enabled] == false -> {:error, :configured_off} @@ -92,9 +93,12 @@ defmodule DevilsDictionary.Artsy.Availability do defp coordinator_enabled?(nil), do: true + defp coordinator_enabled?(coordinator) when is_pid(coordinator), + do: Process.alive?(coordinator) + defp coordinator_enabled?(coordinator) do case Process.whereis(coordinator) do - nil -> true + nil -> false _pid -> DevilsDictionary.Artsy.RequestCoordinator.enabled?(coordinator) end end diff --git a/lib/devils_dictionary/artsy/client.ex b/lib/devils_dictionary/artsy/client.ex index 0c064638..501fe4af 100644 --- a/lib/devils_dictionary/artsy/client.ex +++ b/lib/devils_dictionary/artsy/client.ex @@ -31,6 +31,8 @@ defmodule DevilsDictionary.Artsy.Client do sleep_fun: nil, now_fun: nil, availability_fun: nil, + shared_scope: nil, + shared_request_limit: nil, coordinator: DevilsDictionary.Artsy.RequestCoordinator @type t :: %__MODULE__{} @@ -72,6 +74,9 @@ defmodule DevilsDictionary.Artsy.Client do coordinator ) end, + shared_scope: opts[:shared_scope], + shared_request_limit: + optional_positive!(opts[:shared_request_limit], :shared_request_limit), coordinator: coordinator } end @@ -202,6 +207,7 @@ defmodule DevilsDictionary.Artsy.Client do } end + @doc "Normalizes retained artist fields while keeping opaque ID and slug distinct." def normalize_artist(body) when is_map(body) do %{ "id" => body["id"], @@ -214,6 +220,7 @@ defmodule DevilsDictionary.Artsy.Client do } end + @doc "Normalizes the direct gene identity fields used by versioned meaning mappings." def normalize_gene(body) when is_map(body) do %{"id" => body["id"], "name" => body["name"], "type" => body["type"]} end @@ -339,7 +346,17 @@ defmodule DevilsDictionary.Artsy.Client do client} {:error, failure, client} -> - {:error, %{failure | code: "authentication_unavailable"}, client} + code = + if failure.code in [ + "request_limit", + "shared_request_limit", + "provider_disabled", + "coordinator_unavailable" + ], + do: failure.code, + else: "authentication_unavailable" + + {:error, %{failure | code: code}, client} end end end @@ -385,6 +402,12 @@ defmodule DevilsDictionary.Artsy.Client do else {:error, :provider_disabled} -> {:error, failure(:provider_disabled, nil, URI.parse(url).path), client} + + {:error, :shared_request_limit} -> + {:error, failure(:shared_request_limit, nil, URI.parse(url).path), client} + + {:error, :coordinator_unavailable} -> + {:error, failure(:coordinator_unavailable, nil, URI.parse(url).path), client} end end end @@ -583,7 +606,8 @@ defmodule DevilsDictionary.Artsy.Client do defp redirect_url(response) do case Req.Response.get_header(response, "location") do [url | _] -> url - _ -> get_in(response.body, ["_links", "location", "href"]) + _ when is_map(response.body) -> get_in(response.body, ["_links", "location", "href"]) + _ -> nil end end @@ -622,9 +646,12 @@ defmodule DevilsDictionary.Artsy.Client do defp acquire(%{coordinator: nil}), do: {:ok, 0, 0} defp acquire(client) do - DevilsDictionary.Artsy.RequestCoordinator.acquire(client.coordinator, client.rate_limit_ms) + DevilsDictionary.Artsy.RequestCoordinator.acquire(client.coordinator, client.rate_limit_ms, + scope: client.shared_scope, + limit: client.shared_request_limit + ) catch - :exit, _ -> {:ok, 0, client.rate_limit_ms} + :exit, _ -> {:error, :coordinator_unavailable} end defp generation_current?(%{coordinator: nil}, _generation), do: true @@ -684,4 +711,6 @@ defmodule DevilsDictionary.Artsy.Client do defp positive!(_value, name), do: raise(ArgumentError, "#{name} must be positive") defp non_negative!(value, _name) when is_integer(value) and value >= 0, do: value defp non_negative!(_value, name), do: raise(ArgumentError, "#{name} must be non-negative") + defp optional_positive!(nil, _name), do: nil + defp optional_positive!(value, name), do: positive!(value, name) end diff --git a/lib/devils_dictionary/artsy/request_coordinator.ex b/lib/devils_dictionary/artsy/request_coordinator.ex index d0676f47..e6469499 100644 --- a/lib/devils_dictionary/artsy/request_coordinator.ex +++ b/lib/devils_dictionary/artsy/request_coordinator.ex @@ -11,6 +11,7 @@ defmodule DevilsDictionary.Artsy.RequestCoordinator do @default_name __MODULE__ + @doc false def start_link(opts \\ []) do name = Keyword.get(opts, :name, @default_name) @@ -19,22 +20,32 @@ defmodule DevilsDictionary.Artsy.RequestCoordinator do else: GenServer.start_link(__MODULE__, opts) end - def acquire(server \\ @default_name, interval_ms \\ nil) do - GenServer.call(server, {:acquire, interval_ms}) + @doc "Reserves one globally paced request attempt." + def acquire(server \\ @default_name, interval_ms \\ nil, opts \\ []) do + GenServer.call(server, {:acquire, interval_ms, opts[:scope], opts[:limit]}) end + @doc "Checks that a request generation is still current after an HTTP response." def current?(generation, server \\ @default_name) do GenServer.call(server, {:current?, generation}) end + @doc "Defers every client until at least the supplied number of milliseconds has elapsed." def defer(milliseconds, server \\ @default_name) when is_integer(milliseconds) and milliseconds >= 0 do GenServer.call(server, {:defer, milliseconds}) end + @doc "Atomically disables requests and invalidates responses already in flight." def disable(server \\ @default_name), do: GenServer.call(server, :disable) + + @doc "Re-enables requests without resetting request-attempt accounting." def enable(server \\ @default_name), do: GenServer.call(server, :enable) + + @doc "Returns whether provider requests are currently enabled." def enabled?(server \\ @default_name), do: GenServer.call(server, :enabled?) + + @doc "Returns global attempt, pacing, and scoped-budget counters." def stats(server \\ @default_name), do: GenServer.call(server, :stats) @impl true @@ -49,26 +60,39 @@ defmodule DevilsDictionary.Artsy.RequestCoordinator do next_at: now_fun.(), attempts: 0, interval_ms: Keyword.get(opts, :interval_ms, config[:rate_limit_ms] || 340), + scope_attempts: %{}, now_fun: now_fun }} end @impl true - def handle_call({:acquire, _interval_ms}, _from, %{enabled: false} = state), + def handle_call({:acquire, _interval_ms, _scope, _limit}, _from, %{enabled: false} = state), do: {:reply, {:error, :provider_disabled}, state} - def handle_call({:acquire, interval_ms}, _from, state) do - now = state.now_fun.() - wait_ms = max(state.next_at - now, 0) - interval_ms = if is_integer(interval_ms), do: max(interval_ms, 0), else: state.interval_ms - - state = %{ - state - | next_at: now + wait_ms + interval_ms, - attempts: state.attempts + 1 - } - - {:reply, {:ok, state.generation, wait_ms}, state} + def handle_call({:acquire, interval_ms, scope, limit}, _from, state) do + used = if is_nil(scope), do: 0, else: Map.get(state.scope_attempts, scope, 0) + + if is_integer(limit) and used >= limit do + {:reply, {:error, :shared_request_limit}, state} + else + now = state.now_fun.() + wait_ms = max(state.next_at - now, 0) + interval_ms = if is_integer(interval_ms), do: max(interval_ms, 0), else: state.interval_ms + + scope_attempts = + if is_nil(scope), + do: state.scope_attempts, + else: Map.put(state.scope_attempts, scope, used + 1) + + state = %{ + state + | next_at: now + wait_ms + interval_ms, + attempts: state.attempts + 1, + scope_attempts: scope_attempts + } + + {:reply, {:ok, state.generation, wait_ms}, state} + end end def handle_call({:current?, generation}, _from, state), @@ -85,10 +109,12 @@ defmodule DevilsDictionary.Artsy.RequestCoordinator do end def handle_call(:enable, _from, state), - do: {:reply, :ok, %{state | enabled: true, next_at: 0}} + do: {:reply, :ok, %{state | enabled: true, next_at: state.now_fun.()}} def handle_call(:enabled?, _from, state), do: {:reply, state.enabled, state} def handle_call(:stats, _from, state), - do: {:reply, Map.take(state, [:enabled, :generation, :attempts, :next_at]), state} + do: + {:reply, Map.take(state, [:enabled, :generation, :attempts, :next_at, :scope_attempts]), + state} end diff --git a/lib/devils_dictionary/artworks.ex b/lib/devils_dictionary/artworks.ex index 8eb8a22e..fec3277a 100644 --- a/lib/devils_dictionary/artworks.ex +++ b/lib/devils_dictionary/artworks.ex @@ -91,6 +91,7 @@ defmodule DevilsDictionary.Artworks do } end + @doc "Returns one reusable artwork view by local object identity, or nil." def get(object_id) when is_integer(object_id) do case Repo.one( from entity in Entity, diff --git a/lib/devils_dictionary/artworks/manifest.ex b/lib/devils_dictionary/artworks/manifest.ex index 67e8e009..616c1da4 100644 --- a/lib/devils_dictionary/artworks/manifest.ex +++ b/lib/devils_dictionary/artworks/manifest.ex @@ -4,6 +4,7 @@ defmodule DevilsDictionary.Artworks.Manifest do @schema_version 2 @terminal_statuses ~w(matched created skipped conflict unavailable) + @doc "Creates a portable manifest with every unique exact identity pending." def new(candidates, metadata \\ %{}) when is_list(candidates) do %{ "schema_version" => @schema_version, @@ -22,6 +23,7 @@ defmodule DevilsDictionary.Artworks.Manifest do |> put_checksum() end + @doc "Loads and verifies a supported checksummed manifest." def load!(path) do manifest = path |> File.read!() |> Jason.decode!() @@ -38,6 +40,7 @@ defmodule DevilsDictionary.Artworks.Manifest do upgrade(manifest) end + @doc "Atomically saves a manifest with a refreshed checksum." def save!(manifest, path) do manifest = manifest |> Map.put("updated_at", timestamp()) |> put_checksum() directory = Path.dirname(path) @@ -48,6 +51,7 @@ defmodule DevilsDictionary.Artworks.Manifest do manifest end + @doc "Merges one resumable import outcome into a candidate by index." def update_candidate(manifest, index, outcome) when is_integer(index) and is_map(outcome) do candidates = List.update_at(manifest["candidates"], index, fn candidate -> @@ -58,10 +62,13 @@ defmodule DevilsDictionary.Artworks.Manifest do manifest |> Map.put("candidates", candidates) |> put_checksum() end + @doc "Returns true only for an import outcome that does not need resuming." def completed?(candidate), do: get_in(candidate, ["import", "status"]) in @terminal_statuses + @doc "Lists terminal candidate status strings." def terminal_statuses, do: @terminal_statuses + @doc "Returns the deterministic SHA-256 digest used by manifest verification." def checksum(value) do :sha256 |> :crypto.hash(Jason.encode!(value)) diff --git a/lib/devils_dictionary/artworks/seeder.ex b/lib/devils_dictionary/artworks/seeder.ex index 6c28508e..7d14074c 100644 --- a/lib/devils_dictionary/artworks/seeder.ex +++ b/lib/devils_dictionary/artworks/seeder.ex @@ -31,6 +31,7 @@ defmodule DevilsDictionary.Artworks.Seeder do @max_collection_pages 20 @freshness_seconds 86_400 + @doc "Imports a checksummed candidate manifest with bounded, resumable provider work." def run(manifest, opts \\ []) when is_map(manifest) do dry_run? = Keyword.get(opts, :dry_run, false) resume? = Keyword.get(opts, :resume, false) @@ -205,27 +206,40 @@ defmodule DevilsDictionary.Artworks.Seeder do {:ok, body, client, request_meta} -> artwork = Client.normalize_artwork(body) - payload = - artsy_payload(candidate, artwork) - |> put_in(["request_meta"], stringify(request_meta)) + if valid_provider_artwork?(artwork) do + payload = + artsy_payload(candidate, artwork) + |> put_in(["request_meta"], stringify(request_meta)) - case guarded_store(payload, sources["artsy"], run.id) do - {:ok, record} -> - cached = %{record: record, payload: payload, fresh?: true} + case guarded_store(payload, sources["artsy"], run.id) do + {:ok, record} -> + cached = %{record: record, payload: payload, fresh?: true} - continue_hydrated( - candidate, - work_id, - existing_id, - cached, - client, - sources, - run, - Map.put(stages, "artwork", %{"status" => "complete"}) - ) + continue_hydrated( + candidate, + work_id, + existing_id, + cached, + client, + sources, + run, + Map.put(stages, "artwork", %{"status" => "complete"}) + ) - {:error, :provider_disabled} -> - {partial(work_id, stages, "provider_disabled", :provider_disabled), client} + {:error, :provider_disabled} -> + {partial(work_id, stages, "provider_disabled", :provider_disabled), client} + end + else + {%{ + status: :unavailable, + object_id: work_id, + reason: "artsy_artwork_missing_identity", + stages: + Map.put(stages, "artwork", %{ + "status" => "unavailable", + "error" => "missing_opaque_id_or_slug" + }) + }, client} end {:error, %{code: "not_found"}, client} -> @@ -254,6 +268,19 @@ defmodule DevilsDictionary.Artworks.Seeder do artwork = cached.payload["artwork"] || %{} cond do + not valid_provider_artwork?(artwork) -> + {%{ + status: :unavailable, + object_id: work_id, + source_record_id: cached.record.id, + reason: "artsy_artwork_missing_identity", + stages: + Map.put(stages, "artwork", %{ + "status" => "unavailable", + "error" => "missing_opaque_id_or_slug" + }) + }, client} + artwork["slug"] != candidate["artsy_artwork_slug"] -> entry = artsy_entry(candidate, artwork, cached.record, sources["artsy"], run.id, true) resolution = guarded_resolve(entry) @@ -682,10 +709,26 @@ defmodule DevilsDictionary.Artworks.Seeder do |> Enum.filter(&valid_slug?(&1["artsy_artist_slug"])) |> Map.new(&{&1["artsy_artist_slug"], &1}) + returned_slugs = + artists + |> Enum.map(& &1["slug"]) + |> Enum.filter(&valid_slug?/1) + |> MapSet.new() + + missing_candidates = + Enum.count(candidate["creators"] || [], fn creator -> + slug = creator["artsy_artist_slug"] + not valid_slug?(slug) or not MapSet.member?(returned_slugs, slug) + end) + stats = - Enum.reduce(artists, %{resolved: 0, links: 0, missing: 0}, fn artist, stats -> - case creator_by_slug[artist["slug"]] do - %{"qid" => qid} = creator when is_binary(qid) -> + Enum.reduce(artists, %{resolved: 0, links: 0, missing: missing_candidates}, fn artist, + stats -> + case {valid_provider_artist?(artist), creator_by_slug[artist["slug"]]} do + {false, _creator} -> + %{stats | missing: stats.missing + 1} + + {true, %{"qid" => qid} = creator} when is_binary(qid) -> person_id = Registry.by_external_id("wikidata", qid) if person_id do @@ -722,7 +765,7 @@ defmodule DevilsDictionary.Artworks.Seeder do %{stats | missing: stats.missing + 1} end - _ -> + {_valid?, _creator} -> %{stats | missing: stats.missing + 1} end end) @@ -778,32 +821,42 @@ defmodule DevilsDictionary.Artworks.Seeder do case Repo.get_by(Assertion, source_id: source_id, origin_key: origin_key) do nil -> - case Claims.assert(work_id, "authored_by", person_id, %{ - source_id: source_id, - origin_key: origin_key, - method: "source_relationship", - metadata: %{ - "provider" => provider, - "review" => "attributed_source_fact", - "identity_independent_of_artwork_provider" => provider == "wikidata_manifest" - } - }) do - {:ok, claim} -> - if artwork_record do - revision = Claims.current_revision(claim.id) - - {:ok, _evidence} = - Claims.add_evidence(revision.id, %{ - source_record_revision_id: artwork_record.current_revision.id, - evidence_role: :supports, - attribution_text: "Artsy artwork creator link" - }) - end + result = + Repo.transaction(fn -> + case Claims.assert(work_id, "authored_by", person_id, %{ + source_id: source_id, + origin_key: origin_key, + method: "source_relationship", + metadata: %{ + "provider" => provider, + "review" => "attributed_source_fact", + "identity_independent_of_artwork_provider" => provider == "wikidata_manifest" + } + }) do + {:ok, claim} -> + if artwork_record do + revision = Claims.current_revision(claim.id) + + case Claims.add_evidence(revision.id, %{ + source_record_revision_id: artwork_record.current_revision.id, + evidence_role: :supports, + attribution_text: "Artsy artwork creator link" + }) do + {:ok, _evidence} -> :ok + {:error, reason} -> Repo.rollback(reason) + end + end - true + true + + {:error, reason} -> + Repo.rollback(reason) + end + end) - {:error, _reason} -> - false + case result do + {:ok, true} -> true + {:error, _reason} -> false end _existing -> @@ -891,6 +944,14 @@ defmodule DevilsDictionary.Artworks.Seeder do defp valid_slug?(value), do: is_binary(value) and Regex.match?(~r/\A[a-z0-9][a-z0-9-]{1,254}\z/, value) + defp valid_provider_artwork?(artwork), + do: valid_slug?(artwork["slug"]) and present?(artwork["id"]) + + defp valid_provider_artist?(artist), + do: valid_slug?(artist["slug"]) and present?(artist["id"]) + + defp present?(value), do: is_binary(value) and String.trim(value) != "" + defp dedupe_source_rows(rows) do Enum.uniq_by(rows, fn row -> row["id"] || row["slug"] || :erlang.phash2(row) end) end @@ -963,7 +1024,12 @@ defmodule DevilsDictionary.Artworks.Seeder do defp maybe_save(manifest, nil), do: manifest defp maybe_save(manifest, path), do: Manifest.save!(manifest, path) - defp bounded(value, min, max) when is_integer(value), do: value |> max(min) |> min(max) + + defp bounded(value, min, max) when is_integer(value) and value >= min and value <= max, + do: value + + defp bounded(_value, min, max), + do: raise(ArgumentError, "value is outside the supported bound #{min}..#{max}") defp stringify(map) when is_map(map), do: Map.new(map, fn {key, value} -> {to_string(key), stringify(value)} end) diff --git a/lib/devils_dictionary/artworks/wikidata_candidates.ex b/lib/devils_dictionary/artworks/wikidata_candidates.ex index 9bf11e88..5b4bc523 100644 --- a/lib/devils_dictionary/artworks/wikidata_candidates.ex +++ b/lib/devils_dictionary/artworks/wikidata_candidates.ex @@ -197,8 +197,9 @@ defmodule DevilsDictionary.Artworks.WikidataCandidates do {"user-agent", Application.fetch_env!(:devils_dictionary, :user_agent)} ], receive_timeout: 15_000, - retry: :transient, - max_retries: 2 + # Discovery accounting is per physical request. Disable Req's hidden + # retries so the bounded loop and reported count cannot diverge. + retry: false ] end diff --git a/lib/devils_dictionary_web/live/artwork_live.ex b/lib/devils_dictionary_web/live/artwork_live.ex index 2944dc62..52b7954c 100644 --- a/lib/devils_dictionary_web/live/artwork_live.ex +++ b/lib/devils_dictionary_web/live/artwork_live.ex @@ -24,7 +24,6 @@ defmodule DevilsDictionaryWeb.ArtworkLive do search_form: to_form(%{"q" => ""}, as: :search), artsy_form: to_form(%{"q" => ""}, as: :artsy), artsy_configured: available?, - artsy_requests_used: 0, artsy_status: :idle, artsy_items: [], artsy_query: "", @@ -80,13 +79,15 @@ defmodule DevilsDictionaryWeb.ArtworkLive do query == "" -> {:noreply, assign(socket, artsy_status: :idle, artsy_items: [], artsy_query: "")} - socket.assigns.artsy_requests_used >= 30 -> - {:noreply, assign(socket, artsy_status: :quota, artsy_items: [])} - true -> # The credential-bearing client exists only inside this server task; # it is never retained in LiveView assigns or rendered state. - client = Client.new(request_limit: 30 - socket.assigns.artsy_requests_used) + client = + Client.new( + request_limit: 30, + shared_scope: :interactive_search, + shared_request_limit: 30 + ) {:noreply, socket @@ -101,23 +102,22 @@ defmodule DevilsDictionaryWeb.ArtworkLive do end @impl true - def handle_async(:artsy_search, {:ok, {:ok, result, client}}, socket) do + def handle_async(:artsy_search, {:ok, {:ok, result, _client}}, socket) do items = Enum.map(result.items, &decorate_result/1) {:noreply, assign(socket, - artsy_requests_used: socket.assigns.artsy_requests_used + client.request_count, artsy_status: if(items == [], do: :empty, else: :ready), artsy_items: items )} end - def handle_async(:artsy_search, {:ok, {:error, failure, client}}, socket) do - status = if failure.code == "request_limit", do: :quota, else: :failed + def handle_async(:artsy_search, {:ok, {:error, failure, _client}}, socket) do + status = + if failure.code in ["request_limit", "shared_request_limit"], do: :quota, else: :failed {:noreply, assign(socket, - artsy_requests_used: socket.assigns.artsy_requests_used + client.request_count, artsy_status: status, artsy_items: [] )} diff --git a/lib/devils_dictionary_web/live/connection_live.ex b/lib/devils_dictionary_web/live/connection_live.ex index 6ff7b81f..7d474b8f 100644 --- a/lib/devils_dictionary_web/live/connection_live.ex +++ b/lib/devils_dictionary_web/live/connection_live.ex @@ -214,16 +214,18 @@ defmodule DevilsDictionaryWeb.ConnectionLive do defp preselected_subject(nil), do: nil - defp preselected_subject(value) do + defp preselected_subject(value) when is_binary(value) do case Integer.parse(value) do {id, ""} when id > 0 -> Connection.endpoint(id) _ -> nil end end + defp preselected_subject(_value), do: nil + defp preselected_evidence(nil, _locator), do: {%{}, []} - defp preselected_evidence(value, locator) do + defp preselected_evidence(value, locator) when is_binary(value) do with {id, ""} when id > 0 <- Integer.parse(value), true <- locator != "", true <- @@ -248,6 +250,8 @@ defmodule DevilsDictionaryWeb.ConnectionLive do end end + defp preselected_evidence(_value, _locator), do: {%{}, []} + defp revision_number(nil), do: {:ok, nil} defp revision_number(value) do diff --git a/lib/mix/tasks/dd.artworks.seed.ex b/lib/mix/tasks/dd.artworks.seed.ex index 8c516f78..84d4c8b9 100644 --- a/lib/mix/tasks/dd.artworks.seed.ex +++ b/lib/mix/tasks/dd.artworks.seed.ex @@ -6,21 +6,24 @@ defmodule Mix.Tasks.Dd.Artworks.Seed do identifiers, or imports an existing manifest through the shared source identity resolver. - mix dd.artworks.seed --manifest priv/artworks/manifests/pilot-v1.json --build --dry-run + mix dd.artworks.seed --manifest priv/artworks/manifests/pilot-v1.json --build + mix dd.artworks.seed --manifest priv/artworks/manifests/pilot-v1.json --build --import --dry-run mix dd.artworks.seed --manifest priv/artworks/manifests/pilot-v1.json \ --record-limit 50 --request-limit 160 \ --wikidata-entity-limit 100 --wikidata-request-limit 4 --resume - `--dry-run` never writes the database. `--build` discovers candidates and - writes only the manifest; add `--import` to build and import in one command. + `--build` always makes bounded Wikidata discovery calls and writes the base + manifest. `--dry-run` applies only to the import stage and never writes the + database; add `--import` to build and dry-run the import in one command. Imports hydrate selected artwork and creator QIDs through the existing bounded Wikidata adapter before optional Artsy enrichment. Use `--skip-wikidata-hydration` only when that stage is already complete, and `--refresh-wikidata` when current Wikidata records must be refreshed. - All discovery and provider calls are bounded. The manifest checkpoints every - collection page and identity/creator stage, so `--resume` continues partial - records without replaying completed pages. Reruns converge on exact Wikidata, + All discovery and provider calls are bounded. Import progress is written to + an ignored, environment-local `MANIFEST.checkpoint.json` (or `--checkpoint`), + so `--resume` continues partial records without mutating the portable base + manifest or replaying completed pages. Reruns converge on exact Wikidata, P11005/P2042 and opaque Artsy identifiers; titles never identify. """ @@ -31,6 +34,7 @@ defmodule Mix.Tasks.Dd.Artworks.Seed do @switches [ manifest: :string, + checkpoint: :string, build: :boolean, import: :boolean, dry_run: :boolean, @@ -58,11 +62,18 @@ defmodule Mix.Tasks.Dd.Artworks.Seed do end path = Path.expand(opts[:manifest]) - manifest = if opts[:build], do: build(path, opts), else: Manifest.load!(path) + base_manifest = if opts[:build], do: build(path, opts), else: Manifest.load!(path) if opts[:build] == true and opts[:import] != true do - print_manifest(manifest, path) + print_manifest(base_manifest, path) else + checkpoint_path = Path.expand(opts[:checkpoint] || path <> ".checkpoint.json") + + manifest = + if opts[:resume] && File.exists?(checkpoint_path), + do: Manifest.load!(checkpoint_path), + else: base_manifest + wikidata_stats = if opts[:dry_run] || opts[:skip_wikidata_hydration] do %{skipped: true, reason: if(opts[:dry_run], do: "dry_run", else: "operator_requested")} @@ -74,7 +85,7 @@ defmodule Mix.Tasks.Dd.Artworks.Seed do Seeder.run(manifest, dry_run: opts[:dry_run] || false, resume: opts[:resume] || false, - manifest_path: if(opts[:dry_run], do: nil, else: path), + manifest_path: if(opts[:dry_run], do: nil, else: checkpoint_path), request_limit: opts[:request_limit] || 500, record_limit: opts[:record_limit] || 500, batch_size: opts[:batch_size] || 25 @@ -83,13 +94,14 @@ defmodule Mix.Tasks.Dd.Artworks.Seed do mapping_stats = if opts[:dry_run], do: %{skipped: "dry_run"}, else: Artworks.install_meaning_mappings!() - print_manifest(manifest, path) + print_manifest(manifest, if(opts[:dry_run], do: path, else: checkpoint_path)) + Mix.shell().info("Portable base manifest unchanged: " <> path) Mix.shell().info("Wikidata hydration summary: " <> inspect(wikidata_stats, pretty: true)) Mix.shell().info("Artwork seed summary: " <> inspect(summary, pretty: true)) Mix.shell().info("Meaning mapping summary: " <> inspect(mapping_stats, pretty: true)) end rescue - error in [ArgumentError, File.Error] -> Mix.raise(Exception.message(error)) + error in [ArgumentError, File.Error, Jason.DecodeError] -> Mix.raise(Exception.message(error)) end defp build(path, opts) do @@ -101,6 +113,14 @@ defmodule Mix.Tasks.Dd.Artworks.Seed do remote: not (opts[:local_only] || false) ) + if stats[:error], do: Mix.raise("Wikidata discovery failed: #{stats.error}") + + if stats[:truncated] do + Mix.shell().info( + "Wikidata discovery reached a configured bound; next offset: #{stats.next_offset}" + ) + end + manifest = Manifest.new(candidates, %{ "discovery" => stats, diff --git a/priv/artworks/manifests/pilot-v1.json b/priv/artworks/manifests/pilot-v1.json index 43d84425..f836783c 100644 --- a/priv/artworks/manifests/pilot-v1.json +++ b/priv/artworks/manifests/pilot-v1.json @@ -11,10 +11,7 @@ ], "description": "painting by Anna Rimbaut-Borrel", "import": { - "object_id": 3733926, - "reason": "artsy_artwork_not_found", - "status": "unavailable", - "updated_at": "2026-09-12T18:23:13Z" + "status": "pending" }, "kind": "painting", "qid": "Q106249870", @@ -33,10 +30,7 @@ ], "description": "painting by Paul Klee", "import": { - "object_id": 3733927, - "reason": "artsy_artwork_not_found", - "status": "unavailable", - "updated_at": "2026-09-12T18:23:14Z" + "status": "pending" }, "kind": "painting", "qid": "Q116689343", @@ -55,10 +49,7 @@ ], "description": "painting by Raphael in the National Gallery, London", "import": { - "object_id": 3733928, - "reason": "artsy_artwork_not_found", - "status": "unavailable", - "updated_at": "2026-09-12T18:23:15Z" + "status": "pending" }, "kind": "painting", "qid": "Q11918714", @@ -77,10 +68,7 @@ ], "description": "painting by Titian", "import": { - "object_id": 3733929, - "reason": "artsy_artwork_not_found", - "status": "unavailable", - "updated_at": "2026-09-12T18:23:16Z" + "status": "pending" }, "kind": "painting", "qid": "Q1206860", @@ -99,19 +87,7 @@ ], "description": "painting by Titian", "import": { - "artist_failure": null, - "artsy_artwork_id": "515d39335eeb1c524c0039df", - "artsy_artwork_slug": "titian-cupid-with-the-wheel-of-fortune", - "creator_count": 1, - "creator_links": 0, - "gene_count": 13, - "gene_failure": null, - "missing_creators": 0, - "object_id": 3733930, - "redirects": 1, - "source_record_id": 6984025, - "status": "matched", - "updated_at": "2026-09-12T18:23:18Z" + "status": "pending" }, "kind": "painting", "qid": "Q12115903", @@ -835,9 +811,9 @@ "wikipedia_title": "Christ at the Sea of Galilee" } ], - "checksum": "a8f0a0c01f239e8d54d4f648e37f3d6de30271f0b3f2d7a879c527d0e3e3b398", - "generated_at": "2026-09-12T18:22:45Z", - "schema_version": 1, + "checksum": "8ca842026087267545fee9e3df0740a79247e924fd6dd3e83daa533d04476921", + "generated_at": "2026-09-12T19:52:19Z", + "schema_version": 2, "selection": { "artwork_kinds": [ "painting" @@ -860,5 +836,5 @@ "source": "Wikidata P11005/P2042", "wikipedia_required": false }, - "updated_at": "2026-09-12T18:23:18Z" + "updated_at": "2026-09-12T19:54:47Z" } diff --git a/priv/artworks/manifests/repair-new-entry-v3.json b/priv/artworks/manifests/repair-new-entry-v3.json index 6942e081..b4a555b1 100644 --- a/priv/artworks/manifests/repair-new-entry-v3.json +++ b/priv/artworks/manifests/repair-new-entry-v3.json @@ -11,35 +11,7 @@ ], "description": "1892 painting by Ferdinand Hodler", "import": { - "object_id": 3733934, - "reason": "artsy_artwork_not_found", - "stages": { - "artists": { - "next_cursor": null, - "status": "pending" - }, - "artwork": { - "status": "unavailable" - }, - "creators": { - "status": "pending" - }, - "genes": { - "next_cursor": null, - "status": "pending" - }, - "identity": { - "status": "pending" - }, - "wikidata": { - "preserved": true, - "source_record_id": 6984052, - "source_record_revision_id": 6984052, - "status": "materialized_full_record" - } - }, - "status": "unavailable", - "updated_at": "2026-09-12T19:28:24Z" + "status": "pending" }, "kind": "painting", "qid": "Q122978100", @@ -48,8 +20,8 @@ "wikipedia_title": null } ], - "checksum": "206b3b4691c739f14a9e68416cf57e5aac484db3251fcde4c410af7595e9f381", - "generated_at": "2026-09-12T19:28:05Z", + "checksum": "6ca607c0fe4214e8355994e7113c21f60995f28a4472f5a7928c9e8e8bcb882f", + "generated_at": "2026-09-12T19:52:19Z", "schema_version": 2, "selection": { "artwork_kinds": [ @@ -61,5 +33,5 @@ "source": "issue-86 repair new-entry proof", "wikipedia_required": false }, - "updated_at": "2026-09-12T19:35:20Z" + "updated_at": "2026-09-12T19:54:47Z" } diff --git a/priv/artworks/manifests/repair-pilot-v2.json b/priv/artworks/manifests/repair-pilot-v2.json index d5601400..1ad5870d 100644 --- a/priv/artworks/manifests/repair-pilot-v2.json +++ b/priv/artworks/manifests/repair-pilot-v2.json @@ -11,48 +11,7 @@ ], "description": "painting by Titian", "import": { - "artsy_artwork_id": "515d39335eeb1c524c0039df", - "artsy_artwork_slug": "titian-cupid-with-the-wheel-of-fortune", - "cache_fresh": true, - "creator_count": 1, - "creator_links": 0, - "gene_count": 13, - "missing_creators": 0, - "object_id": 3733930, - "source_record_id": 6984025, - "stages": { - "artists": { - "next_cursor": null, - "pages": 1, - "returned_next_preserved_filter": true, - "status": "complete" - }, - "artwork": { - "status": "complete" - }, - "creators": { - "missing": 0, - "resolved": 1, - "status": "complete" - }, - "genes": { - "next_cursor": null, - "pages": 1, - "returned_next_preserved_filter": true, - "status": "complete" - }, - "identity": { - "status": "complete" - }, - "wikidata": { - "preserved": true, - "source_record_id": 6984024, - "source_record_revision_id": 6984036, - "status": "materialized_full_record" - } - }, - "status": "matched", - "updated_at": "2026-09-12T19:32:52Z" + "status": "pending" }, "kind": "painting", "origin": "local_registry", @@ -72,48 +31,7 @@ ], "description": "painting by Titian", "import": { - "artsy_artwork_id": "515b4aee38ad2d25a7001435", - "artsy_artwork_slug": "titian-ranuccio-farnese-1", - "cache_fresh": true, - "creator_count": 1, - "creator_links": 0, - "gene_count": 15, - "missing_creators": 0, - "object_id": 3733932, - "source_record_id": 6984048, - "stages": { - "artists": { - "next_cursor": null, - "pages": 1, - "returned_next_preserved_filter": true, - "status": "complete" - }, - "artwork": { - "status": "complete" - }, - "creators": { - "missing": 0, - "resolved": 1, - "status": "complete" - }, - "genes": { - "next_cursor": null, - "pages": 1, - "returned_next_preserved_filter": true, - "status": "complete" - }, - "identity": { - "status": "complete" - }, - "wikidata": { - "preserved": true, - "source_record_id": 6984038, - "source_record_revision_id": 6984038, - "status": "materialized_full_record" - } - }, - "status": "matched", - "updated_at": "2026-09-12T19:32:52Z" + "status": "pending" }, "kind": "painting", "origin": "wikidata_sparql", @@ -123,8 +41,8 @@ "wikipedia_title": "Portrait of Ranuccio Farnese" } ], - "checksum": "a86427de706045c3fb9fd3c6defd8e9503ace24dbf7c6722757a731b7fe6fca7", - "generated_at": "2026-09-12T21:30:00Z", + "checksum": "cb0e39e1655c66f87533216dcd827d762a8b759be8416f8d1833f20dbc6cc0ca", + "generated_at": "2026-09-12T19:52:19Z", "schema_version": 2, "selection": { "artwork_kinds": [ @@ -136,5 +54,5 @@ "source": "Wikidata P11005/P2042", "wikipedia_required": false }, - "updated_at": "2026-09-12T19:32:52Z" + "updated_at": "2026-09-12T19:54:47Z" } diff --git a/test/devils_dictionary/artsy/client_test.exs b/test/devils_dictionary/artsy/client_test.exs index ac1257d9..c6772783 100644 --- a/test/devils_dictionary/artsy/client_test.exs +++ b/test/devils_dictionary/artsy/client_test.exs @@ -57,6 +57,19 @@ defmodule DevilsDictionary.Artsy.ClientTest do assert client.retry_count == 1 end + test "terminal 429 remains a quota failure after the bounded retry" do + responses = [ + response(201, %{"token" => "fixture-token"}), + response(429, %{}) |> Req.Response.put_header("retry-after", "0"), + response(429, %{}) |> Req.Response.put_header("retry-after", "0") + ] + + {client, _calls} = client(responses, max_retries: 1) + assert {:error, %{code: "quota_exhausted"}, client} = Client.artwork(client, "work-one") + assert client.request_count == 3 + assert client.retry_count == 1 + end + test "refreshes an expired token once after 401" do responses = [ response(201, %{"token" => "first"}), @@ -82,6 +95,14 @@ defmodule DevilsDictionary.Artsy.ClientTest do assert length(Agent.get(calls, & &1)) == 2 end + test "a redirect with a non-map body and no location fails without crashing" do + {client, _calls} = + client([response(201, %{"token" => "fixture-token"}), response(302, "moved")]) + + assert {:error, %{code: "http_error", status: 302}, _client} = + Client.artwork(client, "work-one") + end + test "fails closed when the request budget is consumed by authentication" do {client, _calls} = client([response(201, %{"token" => "fixture-token"})], request_limit: 1) assert {:error, %{code: "request_limit"}, client} = Client.artwork(client, "work-one") @@ -135,10 +156,83 @@ defmodule DevilsDictionary.Artsy.ClientTest do assert {:ok, _new_generation} = RequestCoordinator.disable(coordinator) refute RequestCoordinator.current?(generation, coordinator) assert {:error, :provider_disabled} = RequestCoordinator.acquire(coordinator, 250) + + assert :ok = RequestCoordinator.enable(coordinator) + assert {:ok, _generation, 0} = RequestCoordinator.acquire(coordinator, 250) + end + + test "shared coordinator enforces one allowance across independent clients" do + coordinator = + start_supervised!({RequestCoordinator, name: nil, interval_ms: 0}, id: make_ref()) + + shared = [coordinator: coordinator, shared_scope: :interactive, shared_request_limit: 2] + + {first, _first_calls} = + client([response(201, %{"token" => "fixture-token"}), artwork_response("work-one")], shared) + + assert {:ok, _body, first, _meta} = Client.artwork(first, "work-one") + assert first.request_count == 2 + + {second, second_calls} = client([response(201, %{"token" => "unused"})], shared) + + assert {:error, %{code: "shared_request_limit"}, second} = + Client.artwork(second, "work-two") + + assert second.request_count == 0 + assert Agent.get(second_calls, & &1) == [] + assert RequestCoordinator.stats(coordinator).scope_attempts == %{interactive: 2} + end + + test "concurrent clients cannot race past a shared allowance" do + coordinator = + start_supervised!({RequestCoordinator, name: nil, interval_ms: 0}, id: make_ref()) + + {client, calls} = + client([artwork_response("work-one")], + coordinator: coordinator, + shared_scope: :interactive, + shared_request_limit: 1 + ) + + client = %{ + client + | token: "fixture-token", + token_expires_at: DateTime.add(DateTime.utc_now(), 3_600, :second) + } + + results = + 1..8 + |> Task.async_stream( + fn index -> Client.artwork(client, "work-#{index}") end, + max_concurrency: 8, + ordered: false, + timeout: :infinity + ) + |> Enum.map(fn {:ok, result} -> result end) + + assert Enum.count(results, &match?({:ok, _, _, _}, &1)) == 1 + + assert Enum.count( + results, + &match?({:error, %{code: "shared_request_limit"}, _}, &1) + ) == 7 + + assert length(Agent.get(calls, & &1)) == 1 + end + + test "a missing coordinator fails closed before authentication leaves the server" do + {client, calls} = + client([response(201, %{"token" => "unused"})], coordinator: :missing_artsy_coordinator) + + assert {:error, %{code: "coordinator_unavailable"}, client} = + Client.artwork(client, "work-one") + + assert client.request_count == 0 + assert Agent.get(calls, & &1) == [] end defp client(responses, opts \\ []) do - calls = start_supervised!({Agent, fn -> [] end}) + calls = start_supervised!({Agent, fn -> [] end}, id: make_ref()) queue = start_supervised!({Agent, fn -> responses end}, id: make_ref()) request_fun = fn options -> diff --git a/test/devils_dictionary/artworks/seeder_test.exs b/test/devils_dictionary/artworks/seeder_test.exs index 1836652e..abbe0784 100644 --- a/test/devils_dictionary/artworks/seeder_test.exs +++ b/test/devils_dictionary/artworks/seeder_test.exs @@ -68,6 +68,25 @@ defmodule DevilsDictionary.Artworks.SeederTest do assert Registry.by_external_id("artsy_artwork_id", "opaque-work-86") == nil end + test "a provider artwork without both opaque id and slug is unavailable and never stored" do + malformed = artwork(nil, "fixture-work", "Painting") + + {manifest, summary} = + Seeder.run(Manifest.new([candidate()]), + artsy_client: client([token(), response(200, malformed)]), + record_limit: 1 + ) + + assert summary.unavailable == 1 + + assert get_in(manifest, ["candidates", Access.at(0), "import", "reason"]) == + "artsy_artwork_missing_identity" + + refute Repo.exists?( + from(record in SourceRecord, where: record.source_id == ^sources_artsy_id()) + ) + end + test "a print/reproduction never collapses into the painting identity" do manifest = Manifest.new([candidate()]) @@ -103,6 +122,37 @@ defmodule DevilsDictionary.Artworks.SeederTest do assert summary.requests == 0 end + @tag :tmp_dir + test "progress is saved to a separate resumable checkpoint without mutating the base manifest", + %{tmp_dir: directory} do + base_path = Path.join(directory, "portable.json") + checkpoint_path = base_path <> ".checkpoint.json" + base = Manifest.save!(Manifest.new([candidate()]), base_path) + base_bytes = File.read!(base_path) + + {_manifest, first} = + Seeder.run(base, + artsy_client: client(unavailable_responses()), + record_limit: 1, + manifest_path: checkpoint_path + ) + + assert first.unavailable == 1 + assert File.exists?(checkpoint_path) + assert File.read!(base_path) == base_bytes + + {_manifest, resumed} = + Seeder.run(Manifest.load!(checkpoint_path), + artsy_client: client([]), + record_limit: 1, + resume: true, + manifest_path: checkpoint_path + ) + + assert resumed.processed == 0 + assert resumed.requests == 0 + end + test "request exhaustion checkpoints artwork and resumes the same manifest without duplicates" do {partial_manifest, partial_summary} = Seeder.run(Manifest.new([candidate()]), @@ -258,6 +308,54 @@ defmodule DevilsDictionary.Artworks.SeederTest do ] end + test "missing and malformed provider artists are counted without losing Wikidata creators" do + candidate = + put_in(candidate()["creators"], [ + %{ + "qid" => "Q900087", + "name" => "Fixture Artist", + "artsy_artist_slug" => "fixture-artist" + }, + %{ + "qid" => "Q900088", + "name" => "Missing Artist", + "artsy_artist_slug" => "missing-artist" + } + ]) + + malformed_artist_page = + response(200, %{ + "_embedded" => %{ + "artists" => [artist(nil, "fixture-artist", "Fixture Artist")] + } + }) + + genes = response(200, %{"_embedded" => %{"genes" => []}}) + + {_manifest, summary} = + Seeder.run(Manifest.new([candidate]), + artsy_client: + client([ + token(), + response(200, artwork("opaque-work-86", "fixture-work", "Painting")), + malformed_artist_page, + genes + ]) + ) + + assert summary.missing_links == 2 + assert summary.creators_resolved == 0 + assert Registry.by_external_id("wikidata", "Q900087") + assert Registry.by_external_id("wikidata", "Q900088") + assert Registry.by_external_id("artsy_artist_id", "") == nil + end + + test "operator limits outside the documented bounds are rejected" do + assert_raise ArgumentError, ~r/supported bound 1\.\.5000/, fn -> + Seeder.run(Manifest.new([candidate()]), record_limit: 0) + end + end + test "withdrawal succeeds after a seeded creator citation and keeps durable QIDs" do {_manifest, _summary} = Seeder.run(Manifest.new([candidate()]), @@ -369,6 +467,11 @@ defmodule DevilsDictionary.Artworks.SeederTest do defp unavailable_responses, do: [token(), response(404, %{})] + defp sources_artsy_id do + %{sources: sources} = Sources.Catalog.seed!() + sources["artsy"].id + end + defp print_responses, do: [token(), response(200, artwork("opaque-print-86", "fixture-work", "Print"))] diff --git a/test/devils_dictionary/artworks/wikidata_candidates_test.exs b/test/devils_dictionary/artworks/wikidata_candidates_test.exs new file mode 100644 index 00000000..fae43dd4 --- /dev/null +++ b/test/devils_dictionary/artworks/wikidata_candidates_test.exs @@ -0,0 +1,29 @@ +defmodule DevilsDictionary.Artworks.WikidataCandidatesTest do + use DevilsDictionary.DataCase, async: false + + alias DevilsDictionary.Artworks.WikidataCandidates + + test "physical discovery requests are counted without hidden Req retries" do + request_fun = fn options -> + send(self(), {:request_options, options}) + + {:ok, + %Req.Response{ + status: 200, + body: %{"results" => %{"bindings" => []}} + }} + end + + assert {:ok, [], %{requests: 1}} = + WikidataCandidates.discover( + limit: 1, + request_limit: 1, + remote: true, + request_fun: request_fun + ) + + assert_receive {:request_options, options} + assert options[:retry] == false + refute Keyword.has_key?(options, :max_retries) + end +end diff --git a/test/devils_dictionary_web/live/artwork_live_test.exs b/test/devils_dictionary_web/live/artwork_live_test.exs index 32beae16..48e004a2 100644 --- a/test/devils_dictionary_web/live/artwork_live_test.exs +++ b/test/devils_dictionary_web/live/artwork_live_test.exs @@ -4,7 +4,7 @@ defmodule DevilsDictionaryWeb.ArtworkLiveTest do import DevilsDictionary.WordFixtures alias DevilsDictionary.{Artworks, Claims, Fixtures, Registry, Repo, Sources} - alias DevilsDictionary.Claims.AssertionEvidence + alias DevilsDictionary.Claims.{Assertion, AssertionEvidence} alias DevilsDictionary.Corpus.SourceRecordRevision alias DevilsDictionary.Sources.{MaterializedOutput, Source} @@ -147,6 +147,7 @@ defmodule DevilsDictionaryWeb.ArtworkLiveTest do ) assert %{installed: 1} = Artworks.install_meaning_mappings!() + assertion_count = Repo.aggregate(Assertion, :count) {:ok, view, _html} = live(ctx.conn, ~p"/define/war") assert has_element?(view, "#artwork-candidates") @@ -157,6 +158,7 @@ defmodule DevilsDictionaryWeb.ArtworkLiveTest do ) assert render(view) =~ "not yet reviewed" + assert Repo.aggregate(Assertion, :count) == assertion_count wrong_id_payload = Map.put(ctx.record.raw, "genes", [ From 2e5fbf736363483571b3e933a9a3595d4d044823 Mon Sep 17 00:00:00 2001 From: Holden Thomas Date: Sat, 12 Sep 2026 21:57:30 +0200 Subject: [PATCH 4/8] Refresh issue 86 verification counts --- docs/audits/2026-09-12-issue86-repair.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/audits/2026-09-12-issue86-repair.md b/docs/audits/2026-09-12-issue86-repair.md index bf689c0b..7eb6bd71 100644 --- a/docs/audits/2026-09-12-issue86-repair.md +++ b/docs/audits/2026-09-12-issue86-repair.md @@ -94,7 +94,8 @@ Artsy ownership without separate evidence/authorization. ## Verification commands -- Focused repository regressions: **48 tests, 0 failures** across the Artsy client, +- Focused repository regressions: **72 tests, 0 failures** across every test file + changed for issue #86, including the Artsy client, seeder, Wikidata adapter and artwork LiveViews. - Original external audit file: **4 of 5 expectations pass**. Its remaining line reads `SourceRecordRevision.payload` directly even though issue #93 intentionally declares @@ -102,7 +103,7 @@ Artsy ownership without separate evidence/authorization. regression uses `Sources.raw/1`, verifies exact payload equality (including P18), and passes. Re-enabling payloads on every ordinary revision query would regress the shared identity foundation's memory-safety fix. -- Full suite through the required `mix precommit`: **934 tests, 0 failures**. The two +- Full suite through the required `mix precommit`: **944 tests, 0 failures**. The two ignored corpus archives were temporarily linked read-only from the original Dictionary checkout so the real source-manifest digests were verified; the links were removed immediately afterward and are not branch artifacts. From 839527a3d95e6f769e2d7ba0f8054d4c9a8b3290 Mon Sep 17 00:00:00 2001 From: Holden Thomas Date: Sun, 13 Sep 2026 11:20:55 +0200 Subject: [PATCH 5/8] Repair issue 86 second independent re-audit findings Addresses every unchecked item in the re-audit at https://github.com/razrfly/dictionary/issues/86#issuecomment-5648331311. Spends zero Artsy HTTP attempts: every proof uses retained provider data, local fixtures, the shared Wikidata adapter, or the browser. Scale, budget and withdrawal: - suggestions query relevant gene assignments first, paginated with no global 100-title or 500-row cutoff; covered by a 501-artwork/501-assignment fixture - the shared interactive request scope renews on a configurable window, kept separate from finite per-import ceilings, tested with an injected clock - queued requests revalidate generation, availability and provider-wide Retry-After deferrals immediately before transmission Provenance-preserving deduplication: - encyclopedia sections group identical semantic relationships for display and counts; the seeder adds evidence to an existing creator claim instead of a parallel one. No assertion or evidence row is deleted. Backfill and reporting: - `--wikidata-only --refresh-wikidata` enriches only identities that already exist locally and reports candidates, identities, images, Commons images, Artsy fallbacks, creator links and unavailable records separately Two defects found by browser verification, each with a regression test that fails without its fix: - image URL and credit fell back independently, so a retained Artsy thumbnail could render under a Wikimedia credit; they are now resolved as a pair - no page passed the contributor flag to a candidate card, so the preselected review composer was unreachable from a definition page Also adds three meaning mappings whose opaque gene IDs were observed directly on retained artwork records, so the catalog produces candidates at all. mix precommit: 954 tests, 0 failures with corpus archives present. Co-Authored-By: Claude Opus 5 --- assets/js/app.js | 3 +- assets/js/artwork_image.mjs | 40 ++++ config/config.exs | 2 + .../2026-09-13-issue86-reaudit-repair.md | 159 ++++++++++++++ docs/integrations/artsy.md | 34 ++- lib/devils_dictionary/artsy/client.ex | 83 ++++--- .../artsy/request_coordinator.ex | 87 ++++++-- lib/devils_dictionary/artworks.ex | 111 ++++++++-- lib/devils_dictionary/artworks/seeder.ex | 65 +++++- .../artworks/wikidata_candidates.ex | 8 +- lib/devils_dictionary/claims/contributions.ex | 38 +++- .../encyclopedia/entity_page.ex | 157 ++++++++++---- .../components/artwork.ex | 9 +- .../live/artwork_live.ex | 7 +- .../live/connection_live.ex | 30 ++- lib/devils_dictionary_web/live/entity_live.ex | 12 +- lib/devils_dictionary_web/live/word_live.ex | 12 ++ lib/mix/tasks/dd.artworks.seed.ex | 161 ++++++++++---- .../manifests/image-fallback-pilot-v4.json | 56 +++++ priv/artworks/meaning-mappings-v1.json | 48 +++++ test/devils_dictionary/artsy/client_test.exs | 94 +++++++- .../artworks/seeder_test.exs | 29 +-- .../artworks/wikidata_candidates_test.exs | 38 ++++ .../live/artwork_live_test.exs | 203 ++++++++++++++++++ 24 files changed, 1285 insertions(+), 201 deletions(-) create mode 100644 assets/js/artwork_image.mjs create mode 100644 docs/audits/2026-09-13-issue86-reaudit-repair.md create mode 100644 priv/artworks/manifests/image-fallback-pilot-v4.json diff --git a/assets/js/app.js b/assets/js/app.js index 31a31213..2cc35b90 100644 --- a/assets/js/app.js +++ b/assets/js/app.js @@ -23,6 +23,7 @@ import "phoenix_html" import {Socket} from "phoenix" import {LiveSocket} from "phoenix_live_view" import {hooks as colocatedHooks} from "phoenix-colocated/devils_dictionary" +import ArtworkImage from "./artwork_image.mjs" import GiphyShelf from "./giphy_shelf.mjs" import topbar from "../vendor/topbar" @@ -30,7 +31,7 @@ const csrfToken = document.querySelector("meta[name='csrf-token']").getAttribute const liveSocket = new LiveSocket("/live", Socket, { longPollFallbackMs: 2500, params: {_csrf_token: csrfToken}, - hooks: {...colocatedHooks, GiphyShelf}, + hooks: {...colocatedHooks, ArtworkImage, GiphyShelf}, }) // Show progress bar on live navigation and form submits diff --git a/assets/js/artwork_image.mjs b/assets/js/artwork_image.mjs new file mode 100644 index 00000000..412d2307 --- /dev/null +++ b/assets/js/artwork_image.mjs @@ -0,0 +1,40 @@ +const ArtworkImage = { + mounted() { + this.image = this.el.querySelector("[data-artwork-image]") + this.fallback = this.el.querySelector("[data-artwork-fallback]") + + this.showImage = () => { + if (this.image) this.image.hidden = false + if (this.fallback) this.fallback.hidden = true + this.el.dataset.imageState = "loaded" + } + + this.showFallback = () => { + if (this.image) this.image.hidden = true + if (this.fallback) this.fallback.hidden = false + this.el.dataset.imageState = "error" + } + + if (!this.image) { + this.showFallback() + this.el.dataset.imageState = "empty" + return + } + + this.image.addEventListener("load", this.showImage) + this.image.addEventListener("error", this.showFallback) + + if (this.image.complete) { + this.image.naturalWidth > 0 ? this.showImage() : this.showFallback() + } + }, + + destroyed() { + if (!this.image) return + + this.image.removeEventListener("load", this.showImage) + this.image.removeEventListener("error", this.showFallback) + }, +} + +export default ArtworkImage diff --git a/config/config.exs b/config/config.exs index ce5f9aea..55b7532d 100644 --- a/config/config.exs +++ b/config/config.exs @@ -136,6 +136,8 @@ config :devils_dictionary, :artsy, endpoint: "https://api.artsy.net", rate_limit_ms: 340, freshness_seconds: 24 * 60 * 60, + interactive_request_limit: 30, + interactive_request_window_ms: 60_000, enabled: true # Oban (#69 §5). `absorb: 1` because a dump absorb is a single long stream. diff --git a/docs/audits/2026-09-13-issue86-reaudit-repair.md b/docs/audits/2026-09-13-issue86-reaudit-repair.md new file mode 100644 index 00000000..e3095063 --- /dev/null +++ b/docs/audits/2026-09-13-issue86-reaudit-repair.md @@ -0,0 +1,159 @@ +# Issue #86 second independent re-audit repair + +Repairs every unchecked item in the independent re-audit at +, which +graded the integration **B — not ready to close** and held PR #98. + +This document records what changed, how each item was verified, and what remains +unproven. It is not an independent re-audit. Nothing here was merged, deployed, +or closed. + +## Provider request accounting + +**This round spent zero Artsy HTTP attempts.** Every repair and every proof below +used retained provider data, local fixtures, the shared Wikidata adapter, or the +browser. The cumulative feasibility campaign therefore remains bounded at **no +more than 194 attempts** (189 documented in the 2026-09-12 repair, plus at most 5 +in the interrupted image-fallback pilot that produced *Rebus* and *Canyon*), +below the issue's 200-request ceiling. No image bytes were downloaded. No +credential or token was printed, logged, or committed; `.env` stays git-ignored. + +## Audit item disposition + +| Audit item | Fix | Verification | +| --- | --- | --- | +| Discovery silently dropping artworks as the catalog grows | `Artworks.suggestions/1` now queries the *relevant* Artsy gene assignments first, paginated by `output_object_id` in 500-row pages with no global cutoff, then loads exactly those identities. The unrelated 100-title catalog clamp is gone. | Deterministic fixture regression with 501 artworks and 501 assignments where only the last matches; it is found. | +| Interactive request allowance never renewing | `RequestCoordinator` tracks a per-scope window start and resets a scope's usage once `window_ms` elapses. `ArtworkLive` reads `interactive_request_limit` / `interactive_request_window_ms` from config (30 per 60s). Per-import finite ceilings are unchanged and separate. | Shared-accounting, exhaustion and recovery test driven by an injected clock. | +| Queued HTTP requests sent after provider disable/withdrawal | After the paced sleep the client revalidates the reservation through `RequestCoordinator.revalidate/2` and rechecks availability and generation immediately before transmission. Provider-wide `Retry-After` deferrals are re-waited by already-queued callers. The existing in-flight response invalidation is preserved. | Mocked regression disabling the provider while a request is queued observes zero transmissions; a second test asserts a queued caller respects a later `Retry-After`. | +| Duplicate visible creator/work relationships | Encyclopedia sections group identical semantic relationships (subject, predicate, object, context, jurisdiction, language, validity) and count the grouped rows, keeping one representative per group. The seeder attaches Artsy evidence to an existing `authored_by` claim instead of creating a parallel one. No assertion or evidence row is deleted. | Browser: Ranuccio shows **1** creator link (was 3); Titian lists **3** works, each once, with a matching count of 3 (was the same two paintings listed six times). Database: 14 active `authored_by` assertions over 9 distinct work/creator pairs, displayed as exactly 9 visible links. | +| Backfill existing sparse records safely | `mix dd.artworks.seed --wikidata-only --refresh-wikidata` hydrates only artwork identities that already exist locally, plus their linked creator QIDs, through the existing bounded Wikidata adapter. It never creates catalog entries and spends no Artsy requests. | Ran twice over the 43-candidate manifest; see the bounded-pilot table below. | +| Verify imagery end to end | An `ArtworkImage` hook marks each card `loading` / `loaded` / `error` / `empty` from the real `load` and `error` events and swaps in the placeholder when decoding fails. Image URL and credit are now resolved **as a pair**, so a retained Artsy thumbnail can never be displayed under an independent Wikimedia credit. | Browser measurements of decoded pixels, below. | +| Prove larger, repeatable seeding | The seed task reports candidates, local identities, images, Commons images, Artsy-thumbnail fallbacks, visible creator links and unavailable records separately. | Bounded pilot and rerun, below; scale proven by fixtures rather than live quota. | +| Prove the word-page flow | A definition page's artwork candidate now carries an internal contributor into the review composer with the exact sense, `illustrates`, the immutable Artsy source-record revision, the opaque gene locator and the rationale preselected. `WordLive` mounts the current scope read-only; `/connect` remains gated server-side. | Full browser walkthrough ending in a submitted claim held at `needs_review`. | + +## Two defects found and fixed during this round's browser verification + +Both were found only because the audit insisted on looking at real rendered +pages rather than at populated fields, and both now carry regression tests that +fail without the fix. + +1. **Misattributed image credit.** `image_url` and `image_attribution` each fell + back to Artsy independently. An entity whose independent image was withheld + but whose credit string survived displayed the **Artsy** thumbnail under a + **Wikimedia Commons** credit. They are now resolved together. +2. **The preselected composer was unreachable.** The rich composer path existed + and was unit-tested, but no page ever rendered a candidate card with the + contributor flag, so the reviewed connection flow could not be entered from a + definition page at all. `WordLive` now passes it. + +## Meaning mappings + +Three mappings were added, raising the enabled set from four to seven. Their +opaque Artsy gene IDs were **verified by direct observation on retained Artsy +artwork records in this repository**, not by new provider probes, so they cost +zero requests: + +| Gene | Gene ID | Exact sense | Relation | +| --- | --- | --- | --- | +| Allegory | `528646d2139b214bd9000687` | `oewn-06893714-n#allegory` — "a visible symbol representing an abstract idea" | related | +| Classical Mythology | `507479c1cd92460002001828` | `oewn-07994846-n#mythology` | related | +| Portrait | `4d90d193dcdd5f44a500007e` | `oewn-03993437-n#portrait` — "any likeness of a person, in any medium" | related | + +Each remains `approved_for_candidate_generation_only` and carries a note saying +what the gene does *not* establish. The six unresolved gene slugs stay disabled. +Before this change no retained artwork carried any enabled gene, so the catalog +produced zero candidates and the word-page flow could not be demonstrated at all. + +## Bounded pilot and duplicate-free rerun + +`mix dd.artworks.seed --manifest priv/artworks/manifests/pilot-v1.json +--wikidata-only --refresh-wikidata --wikidata-request-limit 2`, run twice. + +| Count | Pass 1 | Pass 2 | +| --- | ---: | ---: | +| Manifest candidates | 43 | 43 | +| Candidates with an existing local identity (selected) | 9 | 9 | +| Wikidata HTTP requests | 1 | 1 | +| **Artsy HTTP attempts** | **0** | **0** | +| Wikidata entity records hydrated | 15 | 15 | +| New local identities created | 0 | 0 | +| Unavailable records | 0 | 0 | +| Images | 6 | 6 | +| Commons images | 6 | 6 | +| Artsy-thumbnail fallbacks | 0 | 0 | +| Records with no image | 3 | 3 | +| Visible creator links | 9 | 9 | +| Active `authored_by` assertions | 14 | 14 | +| Distinct work/creator pairs | 9 | 9 | + +Catalog identities, assertion totals and creator links were byte-identical +before, between and after the two passes: the rerun created no duplicate +identity, claim or visible link. + +## Browser acceptance + +Development Chrome against the real development catalog. Decoded pixel +dimensions are reported because a populated URL is not evidence that anything +rendered. Catalog cards are `loading="lazy"`; automated scrolling does not +reliably trigger intersection, so each card's image was forced to fetch and then +measured. + +| Page | Result | +| --- | --- | +| `/artworks` | 9 cards. **6 decode real Commons images** (500×452, 500×604, 500×639, 500×679, 500×607, 500×651); 3 render honest placeholders and are the 3 records with no image. 0 errors. | +| `/define/allegory` | Artwork candidate *Cupid with the Wheel of Time* by Titian, image decoded 500×604, labelled `Related Artsy gene "Allegory" · not yet reviewed` with its caution note, plus links to the work and to Titian. | +| `/entities/.../cupid-with-the-wheel-of-time` | Image decoded 500×604 from Commons, exactly **1** creator link, date, medium, collection, credit and source links. | +| `/entities/.../portrait-of-ranuccio-farnese` | Image decoded 500×607, exactly **1** creator link. | +| `/entities/.../titian` | 3 works, each listed once, count 3. | +| Artsy-thumbnail fallback | Image served from `d32dm0rphc51dk.cloudfront.net`, **decoded 215×260**, credit `Courtesy National Gallery of Art, Washington`, with no Wikimedia credit anywhere on the page. | +| Broken image | State goes `loaded` → `error`, the `` is hidden and the placeholder icon is shown. No broken-image glyph. | +| Provider withdrawal | Commons image still decodes 500×604 and the creator link survives; Artsy-only medium and collection disappear; the definition page drops to **0** artwork candidates. | +| Review composer | Preselected with subject *Cupid with the Wheel of Time*, `illustrates`, sense #308977 `allegory`, source-record revision 6984046 and the gene locator. Submitting produced claim #3856927 at revision 1, review state **`needs_review`**, evidence attached, attributed to the submitting account. | + +### How the two non-default browser states were produced + +Both are reversible development-database demonstrations of real code paths using +real retained provider data. Neither invents provider data. + +- **Artsy-thumbnail fallback**: the independent image projection was removed from + one entity's metadata so the retained Artsy thumbnail became the displayed + image, then restored byte-for-byte from a backup. No live record has an Artsy + thumbnail as its displayed image today, because every Artsy-enriched record in + the catalog also has a Wikidata P18 image, which correctly wins. +- **Provider withdrawal**: `display_allowed` was set to `false` on the three + Artsy source records — the same flag `Artworks.withdraw_artsy/1` sets — then + restored. A destructive withdrawal was deliberately **not** run, because it + permanently deletes retained payloads that cannot be rebuilt without spending + provider requests this campaign no longer has. The destructive path stays + covered by the existing automated withdrawal test. + +## Honest remaining limitations + +- **No live Artsy-thumbnail-only record exists in the catalog.** The fallback is + proven to render and decode a real Artsy CDN image with the correct credit, but + the record used had its independent image temporarily withheld. Two genuine + exact-linked paintings without Wikidata P18 (*Rebus*, *Canyon*) were attempted + in the previous session and both returned unavailable from Artsy. Finding a + retrievable one needs provider requests the 200-request ceiling no longer + comfortably allows. +- **Scale beyond 100 artworks and 500 assignments is proven by fixtures**, not by + a live bulk import. This is deliberate: the audit asked for exactly that, to + avoid spending live quota to prove scale. +- **Reviewed and featured examples remain editorial.** One demonstration claim was + submitted in the development database and left at `needs_review`. Nothing was + approved, and no interpretation was invented. +- **Permanent Artsy retention rights are still not established.** Artsy remains + optional, removable enrichment; independent sources are the durable catalog. +- **The three added mappings are candidate generators, not accepted meanings.** + They widen what a reviewer may consider; they assert nothing. + +## Verification commands + +```sh +mix test test/devils_dictionary/artsy/client_test.exs \ + test/devils_dictionary/artworks \ + test/devils_dictionary_web/live/artwork_live_test.exs +mix precommit +mix dd.artworks.seed --manifest priv/artworks/manifests/pilot-v1.json \ + --wikidata-only --refresh-wikidata --wikidata-request-limit 2 +``` diff --git a/docs/integrations/artsy.md b/docs/integrations/artsy.md index b17bc94c..fdf7f89d 100644 --- a/docs/integrations/artsy.md +++ b/docs/integrations/artsy.md @@ -128,7 +128,7 @@ and an image fallback. The Artsy summary is separately and accurately `created: unavailable: 1`; resuming that manifest also made zero calls and no duplicates. Browser verification used real permitted retained data at desktop and in a narrow -mobile-width Chrome window. It covered the six-work local catalog, local search to +mobile-width Chrome window. It covered the local catalog, local search to one result, the rich *Portrait of Ranuccio Farnese* page, navigation to Titian, and the authenticated review composer preselected with the exact Cupid artwork, exact WordNet `war` sense, `illustrates` predicate, immutable Artsy source-record revision, @@ -142,6 +142,28 @@ comparison. The independent-audit finding ledger, final live manifests, and brow acceptance are in [`2026-09-12-issue86-repair.md`](../audits/2026-09-12-issue86-repair.md). +## Existing-record backfill + +Enrich artwork identities that already exist locally, without adding catalog +entries and without spending Artsy requests: + +```sh +mix dd.artworks.seed \ + --manifest priv/artworks/manifests/pilot-v1.json \ + --wikidata-only --refresh-wikidata --wikidata-request-limit 2 +``` + +It selects only manifest candidates whose QID already resolves to a local +identity, hydrates those plus their linked creator QIDs through the shared +bounded Wikidata adapter, and then prints candidates, local identities, images, +Commons images, Artsy-thumbnail fallbacks, visible creator links and unavailable +records separately. Re-running it is idempotent: the second pass created no new +identity, claim or visible creator link. + +Images and displayed credits are resolved as a pair. A retained Artsy thumbnail +is always shown with Artsy's own rights notice, never under an independent +Wikimedia credit. + ## Retention and shutdown Artsy's [API terms](https://developers.artsy.net/v2/terms) do not provide the @@ -167,8 +189,14 @@ editorial claims remain. The operation is transactional and tested. expected source unavailability, not a reason to weaken identity matching. - Artsy's `gene_id` search filter did not change the sampled result page, so gene traversal/filter acquisition is disabled. Only direct genes from an - exact hydrated artwork are considered. Four exact mappings are enabled; the six - unresolved resource slugs remain documented and disabled. + exact hydrated artwork are considered. Seven exact mappings are enabled; the six + unresolved resource slugs remain documented and disabled. Three of the seven + (Allegory, Classical Mythology, Portrait) use gene IDs observed directly on + retained artwork records, so they cost no provider requests to verify. +- No record in the catalog currently displays an Artsy thumbnail, because every + Artsy-enriched record also has a Wikidata P18 image, which correctly wins. The + fallback is verified by temporarily withholding the independent image in a + development database; see the 2026-09-13 re-audit repair note. - The Metropolitan Museum API remains the better open-image/reference source, but its broad text search is noisy and four audit concepts had no results. - The committed manifest is a reproducible seed set, not an assertion that all diff --git a/lib/devils_dictionary/artsy/client.ex b/lib/devils_dictionary/artsy/client.ex index 501fe4af..804f85da 100644 --- a/lib/devils_dictionary/artsy/client.ex +++ b/lib/devils_dictionary/artsy/client.ex @@ -33,6 +33,7 @@ defmodule DevilsDictionary.Artsy.Client do availability_fun: nil, shared_scope: nil, shared_request_limit: nil, + shared_request_window_ms: nil, coordinator: DevilsDictionary.Artsy.RequestCoordinator @type t :: %__MODULE__{} @@ -77,6 +78,8 @@ defmodule DevilsDictionary.Artsy.Client do shared_scope: opts[:shared_scope], shared_request_limit: optional_positive!(opts[:shared_request_limit], :shared_request_limit), + shared_request_window_ms: + optional_positive!(opts[:shared_request_window_ms], :shared_request_window_ms), coordinator: coordinator } end @@ -372,32 +375,39 @@ defmodule DevilsDictionary.Artsy.Client do true -> with {:ok, generation, wait_ms} <- acquire(client) do client.sleep_fun.(wait_ms) - client = %{client | request_count: client.request_count + 1} - - request_opts = - [ - method: method, - url: url, - redirect: false, - retry: false, - receive_timeout: client.timeout_ms, - connect_options: [timeout: client.timeout_ms] - ] ++ opts - - response = client.request_fun.(request_opts) - - if generation_current?(client, generation) and provider_available(client) == :ok do - handle_response( - response, - client, - method, - url, - opts, - retry_number, - redirects - ) + + with :ok <- await_ready(client, generation), + :ok <- provider_available(client), + true <- generation_current?(client, generation) do + client = %{client | request_count: client.request_count + 1} + + request_opts = + [ + method: method, + url: url, + redirect: false, + retry: false, + receive_timeout: client.timeout_ms, + connect_options: [timeout: client.timeout_ms] + ] ++ opts + + response = client.request_fun.(request_opts) + + if generation_current?(client, generation) and provider_available(client) == :ok do + handle_response( + response, + client, + method, + url, + opts, + retry_number, + redirects + ) + else + {:error, failure(:provider_disabled, nil, URI.parse(url).path), client} + end else - {:error, failure(:provider_disabled, nil, URI.parse(url).path), client} + _ -> {:error, failure(:provider_disabled, nil, URI.parse(url).path), client} end else {:error, :provider_disabled} -> @@ -648,18 +658,37 @@ defmodule DevilsDictionary.Artsy.Client do defp acquire(client) do DevilsDictionary.Artsy.RequestCoordinator.acquire(client.coordinator, client.rate_limit_ms, scope: client.shared_scope, - limit: client.shared_request_limit + limit: client.shared_request_limit, + window_ms: client.shared_request_window_ms ) catch :exit, _ -> {:error, :coordinator_unavailable} end + defp await_ready(%{coordinator: nil}, _generation), do: :ok + + defp await_ready(client, generation) do + case DevilsDictionary.Artsy.RequestCoordinator.revalidate(generation, client.coordinator) do + {:ok, 0} -> + :ok + + {:ok, wait_ms} -> + client.sleep_fun.(wait_ms) + await_ready(client, generation) + + {:error, :provider_disabled} -> + {:error, :provider_disabled} + end + catch + :exit, _ -> {:error, :provider_disabled} + end + defp generation_current?(%{coordinator: nil}, _generation), do: true defp generation_current?(client, generation) do DevilsDictionary.Artsy.RequestCoordinator.current?(generation, client.coordinator) catch - :exit, _ -> true + :exit, _ -> false end defp defer(%{coordinator: nil}, _milliseconds), do: :ok diff --git a/lib/devils_dictionary/artsy/request_coordinator.ex b/lib/devils_dictionary/artsy/request_coordinator.ex index e6469499..115bed1c 100644 --- a/lib/devils_dictionary/artsy/request_coordinator.ex +++ b/lib/devils_dictionary/artsy/request_coordinator.ex @@ -22,7 +22,15 @@ defmodule DevilsDictionary.Artsy.RequestCoordinator do @doc "Reserves one globally paced request attempt." def acquire(server \\ @default_name, interval_ms \\ nil, opts \\ []) do - GenServer.call(server, {:acquire, interval_ms, opts[:scope], opts[:limit]}) + GenServer.call( + server, + {:acquire, interval_ms, opts[:scope], opts[:limit], opts[:window_ms]} + ) + end + + @doc "Revalidates a queued reservation and reports any provider-wide deferral still remaining." + def revalidate(generation, server \\ @default_name) do + GenServer.call(server, {:revalidate, generation}) end @doc "Checks that a request generation is still current after an HTTP response." @@ -61,22 +69,29 @@ defmodule DevilsDictionary.Artsy.RequestCoordinator do attempts: 0, interval_ms: Keyword.get(opts, :interval_ms, config[:rate_limit_ms] || 340), scope_attempts: %{}, + scope_started_at: %{}, + deferred_until: now_fun.(), now_fun: now_fun }} end @impl true - def handle_call({:acquire, _interval_ms, _scope, _limit}, _from, %{enabled: false} = state), - do: {:reply, {:error, :provider_disabled}, state} + def handle_call( + {:acquire, _interval_ms, _scope, _limit, _window_ms}, + _from, + %{enabled: false} = state + ), + do: {:reply, {:error, :provider_disabled}, state} - def handle_call({:acquire, interval_ms, scope, limit}, _from, state) do - used = if is_nil(scope), do: 0, else: Map.get(state.scope_attempts, scope, 0) + def handle_call({:acquire, interval_ms, scope, limit, window_ms}, _from, state) do + now = state.now_fun.() + {used, started_at} = scope_usage(state, scope, window_ms, now) if is_integer(limit) and used >= limit do {:reply, {:error, :shared_request_limit}, state} else - now = state.now_fun.() - wait_ms = max(state.next_at - now, 0) + slot_at = max(max(state.next_at, state.deferred_until), now) + wait_ms = slot_at - now interval_ms = if is_integer(interval_ms), do: max(interval_ms, 0), else: state.interval_ms scope_attempts = @@ -84,11 +99,17 @@ defmodule DevilsDictionary.Artsy.RequestCoordinator do do: state.scope_attempts, else: Map.put(state.scope_attempts, scope, used + 1) + scope_started_at = + if is_nil(scope), + do: state.scope_started_at, + else: Map.put(state.scope_started_at, scope, started_at) + state = %{ state - | next_at: now + wait_ms + interval_ms, + | next_at: slot_at + interval_ms, attempts: state.attempts + 1, - scope_attempts: scope_attempts + scope_attempts: scope_attempts, + scope_started_at: scope_started_at } {:reply, {:ok, state.generation, wait_ms}, state} @@ -98,9 +119,23 @@ defmodule DevilsDictionary.Artsy.RequestCoordinator do def handle_call({:current?, generation}, _from, state), do: {:reply, state.enabled and state.generation == generation, state} + def handle_call({:revalidate, generation}, _from, state) do + if state.enabled and state.generation == generation do + {:reply, {:ok, max(state.deferred_until - state.now_fun.(), 0)}, state} + else + {:reply, {:error, :provider_disabled}, state} + end + end + def handle_call({:defer, milliseconds}, _from, state) do not_before = state.now_fun.() + milliseconds - {:reply, :ok, %{state | next_at: max(state.next_at, not_before)}} + + {:reply, :ok, + %{ + state + | next_at: max(state.next_at, not_before), + deferred_until: max(state.deferred_until, not_before) + }} end def handle_call(:disable, _from, state) do @@ -109,12 +144,38 @@ defmodule DevilsDictionary.Artsy.RequestCoordinator do end def handle_call(:enable, _from, state), - do: {:reply, :ok, %{state | enabled: true, next_at: state.now_fun.()}} + do: + {:reply, :ok, + %{ + state + | enabled: true, + next_at: state.now_fun.(), + deferred_until: state.now_fun.() + }} def handle_call(:enabled?, _from, state), do: {:reply, state.enabled, state} def handle_call(:stats, _from, state), do: - {:reply, Map.take(state, [:enabled, :generation, :attempts, :next_at, :scope_attempts]), - state} + {:reply, + Map.take(state, [ + :enabled, + :generation, + :attempts, + :next_at, + :deferred_until, + :scope_attempts, + :scope_started_at + ]), state} + + defp scope_usage(_state, nil, _window_ms, now), do: {0, now} + + defp scope_usage(state, scope, window_ms, now) do + used = Map.get(state.scope_attempts, scope, 0) + started_at = Map.get(state.scope_started_at, scope, now) + + if is_integer(window_ms) and window_ms > 0 and now - started_at >= window_ms, + do: {0, now}, + else: {used, started_at} + end end diff --git a/lib/devils_dictionary/artworks.ex b/lib/devils_dictionary/artworks.ex index fec3277a..d1711aac 100644 --- a/lib/devils_dictionary/artworks.ex +++ b/lib/devils_dictionary/artworks.ex @@ -24,8 +24,8 @@ defmodule DevilsDictionary.Artworks do @catalog_limit 24 @catalog_max 100 - @candidate_scan_limit 500 - @mapping_path Application.app_dir(:devils_dictionary, "priv/artworks/meaning-mappings-v1.json") + @candidate_page_size 500 + @mapping_file "artworks/meaning-mappings-v1.json" @doc "Searches reusable local artwork identities by title or creator name. No provider call." def search(query \\ "", opts \\ []) do @@ -138,8 +138,8 @@ defmodule DevilsDictionary.Artworks do select: mapping ) - assignments = active_artsy_assignments() - catalog = Map.new(search("", limit: @candidate_scan_limit), &{&1.object_id, &1}) + assignments = active_artsy_assignments(Enum.map(mappings, & &1.parameters["gene_id"])) + catalog = artworks_by_ids(Enum.map(assignments, & &1.object_id)) for sense <- senses, mapping <- mappings, @@ -389,13 +389,14 @@ defmodule DevilsDictionary.Artworks do Enum.map(entities, fn entity -> view = Encyclopedia.view(entity) source = details[entity.object_id] + {image_url, image_attribution} = image_with_credit(view, source) %{ object_id: entity.object_id, title: entity.preferred_label, description: entity.description || (source && source.artwork["description"]), - image_url: view.image_url || (source && source.artwork["thumbnail_url"]), - image_attribution: view.image_attribution || (source && source.artwork["image_rights"]), + image_url: image_url, + image_attribution: image_attribution, qid: view.qid, wikipedia_title: view.wikipedia_title, creators: Map.get(creators, entity.object_id, []), @@ -510,29 +511,95 @@ defmodule DevilsDictionary.Artworks do |> Map.new(&{&1.object_id, &1}) end - defp active_artsy_assignments do - artsy_details( + defp active_artsy_assignments([]), do: [] + + defp active_artsy_assignments(gene_ids) do + gene_ids = gene_ids |> Enum.filter(&is_binary/1) |> Enum.uniq() + paged_artsy_assignments(gene_ids, 0, []) + end + + defp paged_artsy_assignments([], _after_id, accumulated), do: accumulated + + defp paged_artsy_assignments(gene_ids, after_id, accumulated) do + rows = Repo.all( - from details in WorkDetails, + from output in MaterializedOutput, + join: record in SourceRecord, + on: record.id == output.source_record_id and record.display_allowed, + join: source in Source, + on: source.id == record.source_id and source.slug == "artsy" and source.active, + join: revision in SourceRecordRevision, + on: + revision.source_record_id == record.id and + revision.revision_key == record.content_hash, + join: details in WorkDetails, + on: details.entity_id == output.output_object_id and details.work_kind == "artwork", join: object in Object, - on: object.id == details.entity_id and object.lifecycle_state == :active, - where: details.work_kind == "artwork", - order_by: details.entity_id, - limit: @candidate_scan_limit, - select: details.entity_id + on: object.id == output.output_object_id and object.lifecycle_state == :active, + where: is_nil(output.retired_at) and output.output_object_id > ^after_id, + where: + fragment( + "EXISTS (SELECT 1 FROM jsonb_array_elements(COALESCE(?->'genes', '[]'::jsonb)) AS gene WHERE gene->>'id' = ANY(?))", + revision.payload, + type(^gene_ids, {:array, :string}) + ), + distinct: output.output_object_id, + order_by: [asc: output.output_object_id], + limit: ^@candidate_page_size, + select: %{ + object_id: output.output_object_id, + genes: revision.payload["genes"], + source_record_revision_id: revision.id + } ) + + accumulated = accumulated ++ rows + + if length(rows) == @candidate_page_size do + paged_artsy_assignments(gene_ids, List.last(rows).object_id, accumulated) + else + accumulated + end + end + + defp artworks_by_ids([]), do: %{} + + defp artworks_by_ids(object_ids) do + Repo.all( + from entity in Entity, + join: object in Object, + on: object.id == entity.object_id and object.lifecycle_state == :active, + join: details in WorkDetails, + on: details.entity_id == entity.object_id and details.work_kind == "artwork", + where: entity.object_id in ^Enum.uniq(object_ids), + select: entity ) - |> Enum.map(fn {object_id, detail} -> - %{ - object_id: object_id, - genes: detail.genes || [], - source_record_revision_id: detail.source_record_revision_id - } - end) + |> views() + |> Map.new(&{&1.object_id, &1}) end defp meaning_mappings do - @mapping_path |> File.read!() |> Jason.decode!() |> Map.fetch!("mappings") + :devils_dictionary + |> Application.app_dir(Path.join("priv", @mapping_file)) + |> File.read!() + |> Jason.decode!() + |> Map.fetch!("mappings") + end + + # An image and its credit must describe the same picture. Falling back to the + # retained Artsy thumbnail while keeping an independent Wikimedia credit would + # misattribute the displayed image, so the pair is resolved together. + defp image_with_credit(view, source) do + cond do + is_binary(view.image_url) -> + {view.image_url, view.image_attribution} + + is_binary(source && source.artwork["thumbnail_url"]) -> + {source.artwork["thumbnail_url"], source.artwork["image_rights"]} + + true -> + {nil, nil} + end end defp gene_matches?(gene_id, genes) when is_binary(gene_id), diff --git a/lib/devils_dictionary/artworks/seeder.ex b/lib/devils_dictionary/artworks/seeder.ex index 7d14074c..40d239f7 100644 --- a/lib/devils_dictionary/artworks/seeder.ex +++ b/lib/devils_dictionary/artworks/seeder.ex @@ -16,7 +16,7 @@ defmodule DevilsDictionary.Artworks.Seeder do alias DevilsDictionary.Artsy.{Availability, Client} alias DevilsDictionary.Artworks.Manifest alias DevilsDictionary.Claims - alias DevilsDictionary.Claims.Assertion + alias DevilsDictionary.Claims.{Assertion, AssertionEvidence, AssertionRevision} alias DevilsDictionary.Corpus.SourceRecordRevision alias DevilsDictionary.Registry alias DevilsDictionary.Repo @@ -752,7 +752,9 @@ defmodule DevilsDictionary.Artworks.Seeder do %{ stats | resolved: stats.resolved + 1, - links: stats.links + if(linked?, do: 1, else: 0) + links: + stats.links + + if(linked? in [:created, :evidence_added], do: 1, else: 0) } else %{stats | missing: stats.missing + 1} @@ -819,7 +821,7 @@ defmodule DevilsDictionary.Artworks.Seeder do defp ensure_creator_link(work_id, person_id, source_id, artwork_record, provider) do origin_key = "#{provider}:creator:#{work_id}:#{person_id}" - case Repo.get_by(Assertion, source_id: source_id, origin_key: origin_key) do + case semantic_creator_assertion(work_id, person_id) do nil -> result = Repo.transaction(fn -> @@ -847,7 +849,7 @@ defmodule DevilsDictionary.Artworks.Seeder do end end - true + :created {:error, reason} -> Repo.rollback(reason) @@ -855,12 +857,59 @@ defmodule DevilsDictionary.Artworks.Seeder do end) case result do - {:ok, true} -> true - {:error, _reason} -> false + {:ok, :created} -> :created + {:error, _reason} -> :failed end - _existing -> - false + existing -> + add_creator_evidence(existing, artwork_record) + end + end + + defp semantic_creator_assertion(work_id, person_id) do + Repo.one( + from assertion in Assertion, + join: revision in AssertionRevision, + on: revision.assertion_id == assertion.id and revision.is_current, + join: predicate in Claims.Predicate, + on: predicate.id == revision.predicate_id and predicate.key == "authored_by", + where: + revision.subject_object_id == ^work_id and revision.object_object_id == ^person_id and + revision.lifecycle_state == :active, + order_by: [asc: assertion.id], + limit: 1, + select: {assertion, revision} + ) + end + + defp add_creator_evidence(_existing, nil), do: :unchanged + + defp add_creator_evidence({_assertion, revision}, artwork_record) do + revision_id = revision.id + source_revision_id = artwork_record.current_revision.id + + if Repo.exists?( + from evidence in AssertionEvidence, + where: + evidence.assertion_revision_id == ^revision_id and + evidence.source_record_revision_id == ^source_revision_id and + evidence.evidence_role == :supports + ) do + :unchanged + else + case Repo.transaction(fn -> + case Claims.add_evidence(revision_id, %{ + source_record_revision_id: source_revision_id, + evidence_role: :supports, + attribution_text: "Artsy artwork creator link" + }) do + {:ok, _evidence} -> :evidence_added + {:error, reason} -> Repo.rollback(reason) + end + end) do + {:ok, :evidence_added} -> :evidence_added + {:error, _reason} -> :failed + end end end diff --git a/lib/devils_dictionary/artworks/wikidata_candidates.ex b/lib/devils_dictionary/artworks/wikidata_candidates.ex index 5b4bc523..3c8146c6 100644 --- a/lib/devils_dictionary/artworks/wikidata_candidates.ex +++ b/lib/devils_dictionary/artworks/wikidata_candidates.ex @@ -224,14 +224,14 @@ defmodule DevilsDictionary.Artworks.WikidataCandidates do defp normalize_bindings(bindings) do bindings - |> Enum.group_by(&qid(value(&1, "item"))) - |> Enum.reject(fn {qid, _} -> is_nil(qid) end) - |> Enum.map(fn {qid, rows} -> + |> Enum.group_by(&{qid(value(&1, "item")), value(&1, "artsy")}) + |> Enum.reject(fn {{qid, artsy}, _} -> is_nil(qid) or is_nil(artsy) end) + |> Enum.map(fn {{qid, artsy}, rows} -> first = hd(rows) %{ "qid" => qid, - "artsy_artwork_slug" => value(first, "artsy"), + "artsy_artwork_slug" => artsy, "title" => value(first, "itemLabel"), "description" => value(first, "itemDescription"), "wikipedia_title" => wikipedia_title(value(first, "article")), diff --git a/lib/devils_dictionary/claims/contributions.ex b/lib/devils_dictionary/claims/contributions.ex index 1f4370d0..2906c185 100644 --- a/lib/devils_dictionary/claims/contributions.ex +++ b/lib/devils_dictionary/claims/contributions.ex @@ -95,7 +95,7 @@ defmodule DevilsDictionary.Claims.Contributions do claim = unwrap(Claims.assert(subject, predicate, object, attrs)) revision = Claims.current_revision(claim.id) - Enum.each(evidence, &add_exact_evidence!(revision.id, &1, user)) + Enum.each(evidence, &add_exact_evidence!(revision.id, &1, user, subject)) claim end) @@ -593,7 +593,7 @@ defmodule DevilsDictionary.Claims.Contributions do revision = unwrap(Claims.revise(assertion_id, revision_attrs)) copy_evidence!(current.id, revision.id) - Enum.each(evidence, &add_exact_evidence!(revision.id, &1, user)) + Enum.each(evidence, &add_exact_evidence!(revision.id, &1, user, current.subject_object_id)) revision end) end @@ -616,7 +616,7 @@ defmodule DevilsDictionary.Claims.Contributions do end) end - defp add_exact_evidence!(revision_id, attrs, user) do + defp add_exact_evidence!(revision_id, attrs, user, subject_object_id) do attrs = atomize_known(attrs) if attrs[:invalid_target], do: Repo.rollback(:invalid_evidence) @@ -624,10 +624,12 @@ defmodule DevilsDictionary.Claims.Contributions do Map.take(attrs, [ :source_record_revision_id, :content_revision_id, - :sense_revision_id + :sense_revision_id, + :expected_source_slug, + :expected_object_id ]) - validate_evidence_target!(target) + validate_evidence_target!(target, subject_object_id) role = parse_evidence_role(attrs[:evidence_role]) locator = String.trim(attrs[:locator] || "") @@ -635,6 +637,7 @@ defmodule DevilsDictionary.Claims.Contributions do evidence_attrs = target + |> Map.take([:source_record_revision_id, :content_revision_id, :sense_revision_id]) |> Map.put(:evidence_role, role) |> Map.put(:locator, locator) |> Map.put( @@ -645,9 +648,30 @@ defmodule DevilsDictionary.Claims.Contributions do unwrap(Claims.add_evidence(revision_id, evidence_attrs)) end - defp validate_evidence_target!(target) do + defp validate_evidence_target!(target, subject_object_id) do valid? = case target do + %{ + source_record_revision_id: id, + expected_source_slug: source_slug, + expected_object_id: expected_object_id + } + when is_integer(id) and is_binary(source_slug) and is_integer(expected_object_id) -> + expected_object_id == subject_object_id and + Repo.exists?( + from revision in DevilsDictionary.Corpus.SourceRecordRevision, + join: record in DevilsDictionary.Sources.SourceRecord, + on: record.id == revision.source_record_id and record.display_allowed, + join: source in DevilsDictionary.Sources.Source, + on: + source.id == record.source_id and source.slug == ^source_slug and source.active, + join: output in DevilsDictionary.Sources.MaterializedOutput, + on: + output.source_record_id == record.id and + output.output_object_id == ^expected_object_id and is_nil(output.retired_at), + where: revision.id == ^id + ) + %{source_record_revision_id: id} when is_integer(id) -> Repo.exists?(from r in DevilsDictionary.Corpus.SourceRecordRevision, where: r.id == ^id) @@ -783,6 +807,8 @@ defmodule DevilsDictionary.Claims.Contributions do "locator" => :locator, "attribution_text" => :attribution_text, "invalid_target" => :invalid_target, + "expected_source_slug" => :expected_source_slug, + "expected_object_id" => :expected_object_id, "entity_kind" => :entity_kind, "preferred_label" => :preferred_label, "description" => :description, diff --git a/lib/devils_dictionary/encyclopedia/entity_page.ex b/lib/devils_dictionary/encyclopedia/entity_page.ex index e85cc27a..f702e663 100644 --- a/lib/devils_dictionary/encyclopedia/entity_page.ex +++ b/lib/devils_dictionary/encyclopedia/entity_page.ex @@ -40,7 +40,7 @@ defmodule DevilsDictionary.Encyclopedia.EntityPage do import Ecto.Query alias DevilsDictionary.Claims - alias DevilsDictionary.Claims.{Connection, Visibility} + alias DevilsDictionary.Claims.{AssertionRevision, Connection, Visibility} alias DevilsDictionary.Discovery.{Mapping, Result, Run} alias DevilsDictionary.Encyclopedia alias DevilsDictionary.Registry @@ -107,6 +107,7 @@ defmodule DevilsDictionary.Encyclopedia.EntityPage do defp assemble(%Entity{} = entity, opts, identity_state, requested_id, outputs) do id = entity.object_id + entity_details = details(entity) {biography, biography_page} = incoming_page(id, "about", "content", opts[:biography_after]) @@ -132,7 +133,7 @@ defmodule DevilsDictionary.Encyclopedia.EntityPage do other_connections(:incoming, id, opts[:connections_in_after]) {connections_out, connections_out_page} = - other_connections(:outgoing, id, opts[:connections_out_after]) + other_connections(:outgoing, id, opts[:connections_out_after], entity_details) %__MODULE__{ entity: Encyclopedia.view(entity), @@ -142,7 +143,7 @@ defmodule DevilsDictionary.Encyclopedia.EntityPage do canonical_id: entity.object_id, outputs: entity_views(outputs) }, - details: details(entity), + details: entity_details, biography: content_views(Enum.map(biography, & &1.subject_object_id)), works: entity_views(Enum.map(works, & &1.subject_object_id)), definitions: definition_views(Enum.map(definitions, & &1.subject_object_id)), @@ -166,7 +167,7 @@ defmodule DevilsDictionary.Encyclopedia.EntityPage do } end - defp other_connections(direction, object_id, after_cursor) do + defp other_connections(direction, object_id, after_cursor, details \\ %{}) do # The named sections above are all reverse-role sections: biography, # authored works/definitions, editions, and edition contents are incoming # claims. Suppressing the same predicates while walking *out* erased the @@ -175,30 +176,21 @@ defmodule DevilsDictionary.Encyclopedia.EntityPage do # the connection section. filters = case direction do - :incoming -> [exclude_predicates: @presented_predicates] - :outgoing -> [exclude_predicates: ["illustrates"]] - end + :incoming -> + [exclude_predicates: @presented_predicates] - opts = filters ++ [after: after_cursor, limit: @section_cap + 1] + :outgoing -> + exclusions = + if details[:work_kind] == "artwork", + do: ["illustrates", "authored_by"], + else: ["illustrates"] - rows = - case direction do - :incoming -> Claims.incoming(object_id, opts) - :outgoing -> Claims.outgoing(object_id, opts) + [exclude_predicates: exclusions] end - visible = Enum.take(rows, @section_cap) - - count = - case direction do - :incoming -> Claims.count_incoming(object_id, filters) - :outgoing -> Claims.count_outgoing(object_id, filters) - end + opts = filters ++ [after: after_cursor, limit: @section_cap + 1] - page = %{ - count: count, - next: if(length(rows) > @section_cap, do: Claims.next_cursor(visible), else: nil) - } + {visible, page} = semantic_claim_page(direction, object_id, opts) endpoint_ids = Enum.map(visible, fn row -> @@ -227,14 +219,13 @@ defmodule DevilsDictionary.Encyclopedia.EntityPage do end defp meaning_connections(object_id, after_cursor) do - all_rows = - Claims.outgoing(object_id, + {rows, page} = + semantic_claim_page(:outgoing, object_id, predicate: "illustrates", after: after_cursor, limit: @section_cap + 1 ) - rows = Enum.take(all_rows, @section_cap) endpoints = Connection.endpoint_summaries(Enum.map(rows, & &1.object_object_id)) review_states = Claims.display_review_states(Enum.map(rows, & &1.id)) @@ -246,11 +237,7 @@ defmodule DevilsDictionary.Encyclopedia.EntityPage do |> Map.put(:review_state, Map.fetch!(review_states, row.id)) end) - {views, - %{ - count: Claims.count_outgoing(object_id, predicate: "illustrates"), - next: if(length(all_rows) > @section_cap, do: Claims.next_cursor(rows)) - }} + {views, page} end defp discovery_appearances(object_id, after_cursor) do @@ -368,24 +355,110 @@ defmodule DevilsDictionary.Encyclopedia.EntityPage do end defp incoming_page(object_id, predicate, subject_kind, after_cursor) do - filters = [predicate: predicate, subject_kind: subject_kind] + semantic_claim_page(:incoming, object_id, + predicate: predicate, + subject_kind: subject_kind, + after: after_cursor, + limit: @section_cap + 1 + ) + end + + defp semantic_claim_page(direction, object_id, opts) do + family = Registry.canonical_family(object_id) + + query = + case direction do + :incoming -> + where( + AssertionRevision, + [revision], + revision.object_object_id in ^family and revision.is_current and + revision.lifecycle_state == :active + ) + + :outgoing -> + where( + AssertionRevision, + [revision], + revision.subject_object_id in ^family and revision.is_current and + revision.lifecycle_state == :active + ) + end - rows = - Claims.incoming( - object_id, - filters ++ [after: after_cursor, limit: @section_cap + 1] + grouped = + query + |> join(:inner, [revision], predicate in Claims.Predicate, + on: predicate.id == revision.predicate_id, + as: :semantic_predicate ) + |> semantic_predicate(opts[:predicate]) + |> semantic_exclusions(opts[:exclude_predicates]) + |> semantic_subject_kind(opts[:subject_kind]) + |> Claims.visible(:public) + |> group_by([revision], [ + revision.subject_object_id, + revision.predicate_id, + revision.object_object_id, + revision.context_object_id, + revision.jurisdiction_entity_id, + revision.language_tag, + revision.valid_from, + revision.valid_to + ]) + |> select([revision], %{id: min(revision.id)}) - visible = Enum.take(rows, @section_cap) + count = grouped |> subquery() |> Repo.aggregate(:count) - page = %{ - count: Claims.count_incoming(object_id, filters), - next: if(length(rows) > @section_cap, do: Claims.next_cursor(visible), else: nil) - } + grouped = + case opts[:after] do + id when is_integer(id) -> having(grouped, [revision], min(revision.id) > ^id) + _ -> grouped + end + + representative_ids = + grouped + |> order_by([revision], asc: min(revision.id)) + |> limit(^(opts[:limit] || @section_cap + 1)) + |> Repo.all() + |> Enum.map(& &1.id) + + by_id = + Repo.all( + from revision in AssertionRevision, + where: revision.id in ^representative_ids, + preload: :predicate + ) + |> Map.new(&{&1.id, &1}) - {visible, page} + rows = representative_ids |> Enum.map(&Map.fetch!(by_id, &1)) |> Enum.take(@section_cap) + + {rows, + %{ + count: count, + next: + if(length(representative_ids) > @section_cap, + do: List.last(rows).id, + else: nil + ) + }} end + defp semantic_predicate(query, nil), do: query + + defp semantic_predicate(query, key), + do: where(query, [semantic_predicate: predicate], predicate.key == ^key) + + defp semantic_exclusions(query, nil), do: query + defp semantic_exclusions(query, []), do: query + + defp semantic_exclusions(query, keys), + do: where(query, [semantic_predicate: predicate], predicate.key not in ^keys) + + defp semantic_subject_kind(query, nil), do: query + + defp semantic_subject_kind(query, kind), + do: where(query, [revision], revision.subject_kind == ^kind) + # A person's subtype row, a work's, an edition's — whichever this entity has. # One query, and nil for a kind that has no detail table. defp details(%Entity{entity_kind: :person, object_id: id}), diff --git a/lib/devils_dictionary_web/components/artwork.ex b/lib/devils_dictionary_web/components/artwork.ex index 03b8525a..05337859 100644 --- a/lib/devils_dictionary_web/components/artwork.ex +++ b/lib/devils_dictionary_web/components/artwork.ex @@ -17,7 +17,11 @@ defmodule DevilsDictionaryWeb.Artwork do class="group grid grid-cols-[4.5rem_minmax(0,1fr)] gap-4 border-t border-mist-950/10 py-5 first:border-t-0 dark:border-white/10 sm:grid-cols-[6rem_minmax(0,1fr)]" > <.link + id={"#{@id}-image"} navigate={entity_path(@artwork)} + phx-hook="ArtworkImage" + phx-update="ignore" + data-image-state={if(@artwork.image_url, do: "loading", else: "empty")} class="aspect-[4/5] overflow-hidden rounded-sm bg-mist-950/5 transition-transform duration-200 group-hover:-translate-y-0.5 dark:bg-white/5" aria-label={"Open #{@artwork.title}"} > @@ -27,10 +31,12 @@ defmodule DevilsDictionaryWeb.Artwork do alt="" loading="lazy" referrerpolicy="no-referrer" + data-artwork-image class="size-full object-cover" />
diff --git a/lib/mix/tasks/dd.artworks.seed.ex b/lib/mix/tasks/dd.artworks.seed.ex index 84d4c8b9..037d2506 100644 --- a/lib/mix/tasks/dd.artworks.seed.ex +++ b/lib/mix/tasks/dd.artworks.seed.ex @@ -8,6 +8,8 @@ defmodule Mix.Tasks.Dd.Artworks.Seed do mix dd.artworks.seed --manifest priv/artworks/manifests/pilot-v1.json --build mix dd.artworks.seed --manifest priv/artworks/manifests/pilot-v1.json --build --import --dry-run + mix dd.artworks.seed --manifest priv/artworks/manifests/pilot-v1.json \ + --wikidata-only --refresh-wikidata --wikidata-request-limit 2 mix dd.artworks.seed --manifest priv/artworks/manifests/pilot-v1.json \ --record-limit 50 --request-limit 160 \ --wikidata-entity-limit 100 --wikidata-request-limit 4 --resume @@ -17,6 +19,8 @@ defmodule Mix.Tasks.Dd.Artworks.Seed do database; add `--import` to build and dry-run the import in one command. Imports hydrate selected artwork and creator QIDs through the existing bounded Wikidata adapter before optional Artsy enrichment. Use + `--wikidata-only --refresh-wikidata` for a safe, repeatable rich-data backfill + of existing identities without spending Artsy requests. Use `--skip-wikidata-hydration` only when that stage is already complete, and `--refresh-wikidata` when current Wikidata records must be refreshed. @@ -37,6 +41,7 @@ defmodule Mix.Tasks.Dd.Artworks.Seed do checkpoint: :string, build: :boolean, import: :boolean, + wikidata_only: :boolean, dry_run: :boolean, resume: :boolean, local_only: :boolean, @@ -64,41 +69,65 @@ defmodule Mix.Tasks.Dd.Artworks.Seed do path = Path.expand(opts[:manifest]) base_manifest = if opts[:build], do: build(path, opts), else: Manifest.load!(path) - if opts[:build] == true and opts[:import] != true do - print_manifest(base_manifest, path) - else - checkpoint_path = Path.expand(opts[:checkpoint] || path <> ".checkpoint.json") - - manifest = - if opts[:resume] && File.exists?(checkpoint_path), - do: Manifest.load!(checkpoint_path), - else: base_manifest - - wikidata_stats = - if opts[:dry_run] || opts[:skip_wikidata_hydration] do - %{skipped: true, reason: if(opts[:dry_run], do: "dry_run", else: "operator_requested")} - else - hydrate_wikidata(manifest, opts) - end - - {manifest, summary} = - Seeder.run(manifest, - dry_run: opts[:dry_run] || false, - resume: opts[:resume] || false, - manifest_path: if(opts[:dry_run], do: nil, else: checkpoint_path), - request_limit: opts[:request_limit] || 500, - record_limit: opts[:record_limit] || 500, - batch_size: opts[:batch_size] || 25 + cond do + opts[:build] == true and opts[:import] != true and opts[:wikidata_only] != true -> + print_manifest(base_manifest, path) + + opts[:wikidata_only] == true -> + if opts[:dry_run], do: Mix.raise("--wikidata-only cannot be combined with --dry-run") + + wikidata_stats = hydrate_wikidata(base_manifest, opts, existing_only: true) + mapping_stats = Artworks.install_meaning_mappings!() + print_manifest(base_manifest, path) + Mix.shell().info("Wikidata backfill summary: " <> inspect(wikidata_stats, pretty: true)) + + Mix.shell().info( + "Selected catalog coverage: " <> inspect(coverage(base_manifest), pretty: true) ) - mapping_stats = - if opts[:dry_run], do: %{skipped: "dry_run"}, else: Artworks.install_meaning_mappings!() + Mix.shell().info("Meaning mapping summary: " <> inspect(mapping_stats, pretty: true)) + + true -> + checkpoint_path = Path.expand(opts[:checkpoint] || path <> ".checkpoint.json") + + manifest = + if opts[:resume] && File.exists?(checkpoint_path), + do: Manifest.load!(checkpoint_path), + else: base_manifest + + wikidata_stats = + if opts[:dry_run] || opts[:skip_wikidata_hydration] do + %{ + skipped: true, + reason: if(opts[:dry_run], do: "dry_run", else: "operator_requested") + } + else + hydrate_wikidata(manifest, opts) + end + + {manifest, summary} = + Seeder.run(manifest, + dry_run: opts[:dry_run] || false, + resume: opts[:resume] || false, + manifest_path: if(opts[:dry_run], do: nil, else: checkpoint_path), + request_limit: opts[:request_limit] || 500, + record_limit: opts[:record_limit] || 500, + batch_size: opts[:batch_size] || 25 + ) + + mapping_stats = + if opts[:dry_run], do: %{skipped: "dry_run"}, else: Artworks.install_meaning_mappings!() + + print_manifest(manifest, if(opts[:dry_run], do: path, else: checkpoint_path)) + Mix.shell().info("Portable base manifest unchanged: " <> path) + Mix.shell().info("Wikidata hydration summary: " <> inspect(wikidata_stats, pretty: true)) + Mix.shell().info("Artwork seed summary: " <> inspect(summary, pretty: true)) + + Mix.shell().info( + "Selected catalog coverage: " <> inspect(coverage(manifest), pretty: true) + ) - print_manifest(manifest, if(opts[:dry_run], do: path, else: checkpoint_path)) - Mix.shell().info("Portable base manifest unchanged: " <> path) - Mix.shell().info("Wikidata hydration summary: " <> inspect(wikidata_stats, pretty: true)) - Mix.shell().info("Artwork seed summary: " <> inspect(summary, pretty: true)) - Mix.shell().info("Meaning mapping summary: " <> inspect(mapping_stats, pretty: true)) + Mix.shell().info("Meaning mapping summary: " <> inspect(mapping_stats, pretty: true)) end rescue error in [ArgumentError, File.Error, Jason.DecodeError] -> Mix.raise(Exception.message(error)) @@ -137,27 +166,71 @@ defmodule Mix.Tasks.Dd.Artworks.Seed do ) end - defp hydrate_wikidata(manifest, opts) do + defp hydrate_wikidata(manifest, opts, hydrate_opts \\ []) do + existing_only? = hydrate_opts[:existing_only] || false + qids = manifest["candidates"] + |> Enum.filter(fn candidate -> + not existing_only? or + not is_nil(DevilsDictionary.Registry.by_external_id("wikidata", candidate["qid"])) + end) |> Enum.flat_map(fn candidate -> [candidate["qid"] | Enum.map(candidate["creators"] || [], & &1["qid"])] end) |> Enum.filter(&is_binary/1) |> Enum.uniq() - entity_limit = opts[:wikidata_entity_limit] || max(length(qids), 1) - request_limit = opts[:wikidata_request_limit] || max(div(length(qids) + 49, 50), 1) + if qids == [] do + %{skipped: true, reason: "no_matching_local_identities", requests: 0, records: 0} + else + entity_limit = opts[:wikidata_entity_limit] || length(qids) + request_limit = opts[:wikidata_request_limit] || max(div(length(qids) + 49, 50), 1) + + {:ok, stats} = + DevilsDictionary.Absorb.Sources.Wikidata.absorb(nil, + qids: Enum.take(qids, entity_limit), + entity_budget: entity_limit, + request_budget: request_limit, + related_depth: 1, + refresh: opts[:refresh_wikidata] || false + ) - {:ok, stats} = - DevilsDictionary.Absorb.Sources.Wikidata.absorb(nil, - qids: Enum.take(qids, entity_limit), - entity_budget: entity_limit, - request_budget: request_limit, - related_depth: 1, - refresh: opts[:refresh_wikidata] || false - ) + stats + end + end - stats + defp coverage(manifest) do + artworks = + manifest["candidates"] + |> Enum.map(&DevilsDictionary.Registry.by_external_id("wikidata", &1["qid"])) + |> Enum.reject(&is_nil/1) + |> Enum.uniq() + |> Enum.map(&Artworks.get/1) + |> Enum.reject(&is_nil/1) + + %{ + candidates: length(manifest["candidates"]), + local_identities: length(artworks), + images: Enum.count(artworks, &is_binary(&1.image_url)), + commons_images: + Enum.count(artworks, &(&1.image_url && String.contains?(&1.image_url, "wikimedia"))), + artsy_thumbnail_fallbacks: + Enum.count( + artworks, + &(&1.artsy && &1.image_url == &1.artsy["thumbnail_url"] && is_binary(&1.image_url)) + ), + visible_creator_links: + artworks + |> Enum.flat_map(fn artwork -> + Enum.map(artwork.creators, &{artwork.object_id, &1.object_id}) + end) + |> Enum.uniq() + |> length(), + unavailable_records: + Enum.count(manifest["candidates"], fn candidate -> + get_in(candidate, ["import", "status"]) == "unavailable" + end) + } end end diff --git a/priv/artworks/manifests/image-fallback-pilot-v4.json b/priv/artworks/manifests/image-fallback-pilot-v4.json new file mode 100644 index 00000000..62731b7b --- /dev/null +++ b/priv/artworks/manifests/image-fallback-pilot-v4.json @@ -0,0 +1,56 @@ +{ + "candidates": [ + { + "artsy_artwork_slug": "robert-rauschenberg-rebus", + "creators": [ + { + "artsy_artist_slug": "robert-rauschenberg", + "name": "Robert Rauschenberg", + "qid": "Q164358" + } + ], + "description": "painting by Robert Rauschenberg", + "import": { + "status": "pending" + }, + "kind": "painting", + "qid": "Q19887272", + "selection_reason": "wikidata_p11005_painting", + "title": "Rebus", + "wikipedia_title": null + }, + { + "artsy_artwork_slug": "robert-rauschenberg-canyon", + "creators": [ + { + "artsy_artist_slug": "robert-rauschenberg", + "name": "Robert Rauschenberg", + "qid": "Q164358" + } + ], + "description": "painting by Robert Rauschenberg", + "import": { + "status": "pending" + }, + "kind": "painting", + "qid": "Q19887473", + "selection_reason": "wikidata_p11005_with_english_wikipedia", + "title": "Canyon", + "wikipedia_title": "Canyon (Rauschenberg)" + } + ], + "checksum": "26342f93867a844f86461539ba9524dc7459b19762d65a0f8febf242b2f06e4f", + "generated_at": "2026-09-12T20:16:10Z", + "schema_version": 2, + "selection": { + "artwork_kinds": [ + "painting" + ], + "identity": "Wikidata QID plus P11005; title is never identity", + "parent_manifest_checksum": "8ca842026087267545fee9e3df0740a79247e924fd6dd3e83daa533d04476921", + "pilot_cap": 500, + "selection_reason": "exact-linked paintings without Wikidata P18 for thumbnail fallback verification", + "source": "Wikidata P11005/P2042", + "wikipedia_required": false + } +} diff --git a/priv/artworks/meaning-mappings-v1.json b/priv/artworks/meaning-mappings-v1.json index 31fbc1d0..0c354ce7 100644 --- a/priv/artworks/meaning-mappings-v1.json +++ b/priv/artworks/meaning-mappings-v1.json @@ -66,6 +66,54 @@ "mapping_review_basis": "Exact Artsy gene resource and exact Open English WordNet source sense were verified; semantic use remains related and requires per-artwork review.", "note": "The provider subject is related evidence only; a reviewer must distinguish affection, romance, attachment, and other senses." }, + { + "version": 2, + "enabled": true, + "sense_source_slug": "wordnet", + "sense_external_key": "oewn-06893714-n#allegory", + "lemma": "allegory", + "language": "en", + "gene_id": "528646d2139b214bd9000687", + "gene_name": "Allegory", + "match_type": "related", + "mapping_review_status": "approved_for_candidate_generation_only", + "mapping_review_actor": "issue-86 bounded feasibility review", + "mapping_reviewed_at": "2026-09-13", + "mapping_review_basis": "The opaque Artsy gene ID was verified by direct observation on a retained Artsy artwork record in this repository's bounded pilot, not by an additional provider probe, and the exact Open English WordNet source sense was verified locally. Semantic use remains related and requires per-artwork review.", + "note": "Allegory is related subject vocabulary for the visible-symbol sense. It does not establish which abstract idea a work personifies; a reviewer must inspect the depicted figures." + }, + { + "version": 2, + "enabled": true, + "sense_source_slug": "wordnet", + "sense_external_key": "oewn-07994846-n#mythology", + "lemma": "mythology", + "language": "en", + "gene_id": "507479c1cd92460002001828", + "gene_name": "Classical Mythology", + "match_type": "related", + "mapping_review_status": "approved_for_candidate_generation_only", + "mapping_review_actor": "issue-86 bounded feasibility review", + "mapping_reviewed_at": "2026-09-13", + "mapping_review_basis": "The opaque Artsy gene ID was verified by direct observation on a retained Artsy artwork record in this repository's bounded pilot, not by an additional provider probe, and the exact Open English WordNet source sense was verified locally. Semantic use remains related and requires per-artwork review.", + "note": "Classical Mythology is related subject vocabulary for the collective-stories sense. It does not identify which myth a work depicts; a reviewer must confirm the specific narrative." + }, + { + "version": 2, + "enabled": true, + "sense_source_slug": "wordnet", + "sense_external_key": "oewn-03993437-n#portrait", + "lemma": "portrait", + "language": "en", + "gene_id": "4d90d193dcdd5f44a500007e", + "gene_name": "Portrait", + "match_type": "related", + "mapping_review_status": "approved_for_candidate_generation_only", + "mapping_review_actor": "issue-86 bounded feasibility review", + "mapping_reviewed_at": "2026-09-13", + "mapping_review_basis": "The opaque Artsy gene ID was verified by direct observation on a retained Artsy artwork record in this repository's bounded pilot, not by an additional provider probe, and the exact Open English WordNet source sense was verified locally. Semantic use remains related and requires per-artwork review.", + "note": "Portrait is related subject vocabulary for the likeness sense. It does not establish the sitter's identity; a reviewer must confirm who is depicted." + }, { "version": 2, "enabled": false, diff --git a/test/devils_dictionary/artsy/client_test.exs b/test/devils_dictionary/artsy/client_test.exs index c6772783..65f45813 100644 --- a/test/devils_dictionary/artsy/client_test.exs +++ b/test/devils_dictionary/artsy/client_test.exs @@ -183,6 +183,26 @@ defmodule DevilsDictionary.Artsy.ClientTest do assert RequestCoordinator.stats(coordinator).scope_attempts == %{interactive: 2} end + test "shared allowances renew independently from finite client import ceilings" do + clock = start_supervised!({Agent, fn -> 5_000 end}, id: make_ref()) + + coordinator = + start_supervised!( + {RequestCoordinator, + name: nil, interval_ms: 0, now_fun: fn -> Agent.get(clock, & &1) end}, + id: make_ref() + ) + + opts = [scope: :interactive, limit: 2, window_ms: 1_000] + assert {:ok, generation, 0} = RequestCoordinator.acquire(coordinator, 0, opts) + assert {:ok, ^generation, 0} = RequestCoordinator.acquire(coordinator, 0, opts) + assert {:error, :shared_request_limit} = RequestCoordinator.acquire(coordinator, 0, opts) + + Agent.update(clock, &(&1 + 1_000)) + assert {:ok, ^generation, 0} = RequestCoordinator.acquire(coordinator, 0, opts) + assert RequestCoordinator.stats(coordinator).scope_attempts == %{interactive: 1} + end + test "concurrent clients cannot race past a shared allowance" do coordinator = start_supervised!({RequestCoordinator, name: nil, interval_ms: 0}, id: make_ref()) @@ -231,6 +251,63 @@ defmodule DevilsDictionary.Artsy.ClientTest do assert Agent.get(calls, & &1) == [] end + test "a queued request is cancelled before transmission when the provider is disabled" do + coordinator = + start_supervised!({RequestCoordinator, name: nil, interval_ms: 50}, id: make_ref()) + + assert {:ok, _generation, 0} = RequestCoordinator.acquire(coordinator, 50) + + sleep_fun = fn wait_ms -> + if wait_ms > 0, do: RequestCoordinator.disable(coordinator) + :ok + end + + {client, calls} = + client([response(201, %{"token" => "unused"})], + coordinator: coordinator, + sleep_fun: sleep_fun + ) + + assert {:error, %{code: "provider_disabled"}, client} = + Client.artwork(client, "work-one") + + assert client.request_count == 0 + assert Agent.get(calls, & &1) == [] + end + + test "Retry-After deferrals delay clients that already share the coordinator" do + clock = start_supervised!({Agent, fn -> 0 end}, id: make_ref()) + sleeps = start_supervised!({Agent, fn -> [] end}, id: make_ref()) + + coordinator = + start_supervised!( + {RequestCoordinator, + name: nil, interval_ms: 0, now_fun: fn -> Agent.get(clock, & &1) end}, + id: make_ref() + ) + + quota = response(429, %{}) |> Req.Response.put_header("retry-after", "2") + + {first, _calls} = + client([response(201, %{"token" => "fixture-token"}), quota], coordinator: coordinator) + + assert {:error, %{code: "quota_exhausted"}, _client} = Client.artwork(first, "work-one") + + sleep_fun = fn milliseconds -> + Agent.update(sleeps, &[milliseconds | &1]) + Agent.update(clock, &(&1 + milliseconds)) + end + + {second, _calls} = + client([response(201, %{"token" => "second-token"}), artwork_response("work-two")], + coordinator: coordinator, + sleep_fun: sleep_fun + ) + + assert {:ok, _body, _client, _meta} = Client.artwork(second, "work-two") + assert 2_000 in Agent.get(sleeps, & &1) + end + defp client(responses, opts \\ []) do calls = start_supervised!({Agent, fn -> [] end}, id: make_ref()) queue = start_supervised!({Agent, fn -> responses end}, id: make_ref()) @@ -246,13 +323,16 @@ defmodule DevilsDictionary.Artsy.ClientTest do client = Client.new( - [ - client_id: "fixture-id", - client_secret: "fixture-secret", - request_fun: request_fun, - sleep_fun: fn _ -> :ok end, - rate_limit_ms: 0 - ] ++ opts + Keyword.merge( + [ + client_id: "fixture-id", + client_secret: "fixture-secret", + request_fun: request_fun, + sleep_fun: fn _ -> :ok end, + rate_limit_ms: 0 + ], + opts + ) ) {client, calls} diff --git a/test/devils_dictionary/artworks/seeder_test.exs b/test/devils_dictionary/artworks/seeder_test.exs index abbe0784..edd8a8c1 100644 --- a/test/devils_dictionary/artworks/seeder_test.exs +++ b/test/devils_dictionary/artworks/seeder_test.exs @@ -69,18 +69,21 @@ defmodule DevilsDictionary.Artworks.SeederTest do end test "a provider artwork without both opaque id and slug is unavailable and never stored" do - malformed = artwork(nil, "fixture-work", "Painting") - - {manifest, summary} = - Seeder.run(Manifest.new([candidate()]), - artsy_client: client([token(), response(200, malformed)]), - record_limit: 1 - ) - - assert summary.unavailable == 1 - - assert get_in(manifest, ["candidates", Access.at(0), "import", "reason"]) == - "artsy_artwork_missing_identity" + for malformed <- [ + artwork(nil, "fixture-work", "Painting"), + artwork("opaque-work-86", nil, "Painting") + ] do + {manifest, summary} = + Seeder.run(Manifest.new([candidate()]), + artsy_client: client([token(), response(200, malformed)]), + record_limit: 1 + ) + + assert summary.unavailable == 1 + + assert get_in(manifest, ["candidates", Access.at(0), "import", "reason"]) == + "artsy_artwork_missing_identity" + end refute Repo.exists?( from(record in SourceRecord, where: record.source_id == ^sources_artsy_id()) @@ -367,7 +370,7 @@ defmodule DevilsDictionary.Artworks.SeederTest do artist_id = Registry.by_external_id("wikidata", "Q900087") assert {:ok, summary} = Artworks.withdraw_artsy("fixture provider shutdown") - assert summary.claims_withdrawn == 1 + assert summary.claims_withdrawn == 0 assert summary.evidence_removed == 1 # Artwork collection checkpoints are immutable revisions too; withdrawal # removes the entire provider-owned history, not only its final snapshot. diff --git a/test/devils_dictionary/artworks/wikidata_candidates_test.exs b/test/devils_dictionary/artworks/wikidata_candidates_test.exs index fae43dd4..3cbf680f 100644 --- a/test/devils_dictionary/artworks/wikidata_candidates_test.exs +++ b/test/devils_dictionary/artworks/wikidata_candidates_test.exs @@ -26,4 +26,42 @@ defmodule DevilsDictionary.Artworks.WikidataCandidatesTest do assert options[:retry] == false refute Keyword.has_key?(options, :max_retries) end + + test "distinct P11005 values on one QID remain distinct artwork identities" do + request_fun = fn _options -> + {:ok, + %Req.Response{ + status: 200, + body: %{ + "results" => %{ + "bindings" => [ + binding("Q900100", "first-artsy-work"), + binding("Q900100", "second-artsy-work") + ] + } + } + }} + end + + assert {:ok, candidates, %{requests: 1}} = + WikidataCandidates.discover( + limit: 2, + request_limit: 1, + remote: true, + request_fun: request_fun + ) + + assert Enum.map(candidates, & &1["artsy_artwork_slug"]) == [ + "first-artsy-work", + "second-artsy-work" + ] + end + + defp binding(qid, artsy_slug) do + %{ + "item" => %{"value" => "http://www.wikidata.org/entity/#{qid}"}, + "itemLabel" => %{"value" => "Fixture artwork"}, + "artsy" => %{"value" => artsy_slug} + } + end end diff --git a/test/devils_dictionary_web/live/artwork_live_test.exs b/test/devils_dictionary_web/live/artwork_live_test.exs index 48e004a2..f56cc860 100644 --- a/test/devils_dictionary_web/live/artwork_live_test.exs +++ b/test/devils_dictionary_web/live/artwork_live_test.exs @@ -6,6 +6,7 @@ defmodule DevilsDictionaryWeb.ArtworkLiveTest do alias DevilsDictionary.{Artworks, Claims, Fixtures, Registry, Repo, Sources} alias DevilsDictionary.Claims.{Assertion, AssertionEvidence} alias DevilsDictionary.Corpus.SourceRecordRevision + alias DevilsDictionary.Encyclopedia.EntityPage alias DevilsDictionary.Sources.{MaterializedOutput, Source} setup ctx do @@ -75,6 +76,16 @@ defmodule DevilsDictionaryWeb.ArtworkLiveTest do {:ok, view, _html} = live(ctx.conn, ~p"/artworks") assert has_element?(view, "#artwork-#{ctx.work.object_id}") + assert has_element?( + view, + "#artwork-#{ctx.work.object_id}-image[phx-hook='ArtworkImage'][data-image-state='loading'] img[data-artwork-image]" + ) + + assert has_element?( + view, + "#artwork-#{ctx.work.object_id}-image [data-artwork-fallback][hidden]" + ) + assert has_element?( view, "#artwork-#{ctx.work.object_id} a[href^='/entities/#{ctx.work.object_id}/']" @@ -203,6 +214,8 @@ defmodule DevilsDictionaryWeb.ArtworkLiveTest do ) assert has_element?(view, "#artwork-metadata") + assert has_element?(view, "#entity-artwork[phx-hook='ArtworkImage'] img[data-artwork-image]") + assert has_element?(view, "#entity-artwork [data-artwork-fallback][hidden]") assert has_element?(view, "#artwork-metadata a[href^='/entities/#{ctx.creator.object_id}/']") assert render(view) =~ "Oil on canvas" assert render(view) =~ "Fixture Museum" @@ -235,6 +248,89 @@ defmodule DevilsDictionaryWeb.ArtworkLiveTest do refute ctx.work.object_id == sense.object_id end + test "composer ignores non-text and unrelated preselected evidence", ctx do + %{conn: conn, user: user} = register_and_log_in_user(%{conn: ctx.conn}) + _user = Repo.update!(Ecto.Changeset.change(user, internal_contributor: true)) + + {:ok, view, _html} = + live( + conn, + "/connect?rationale[]=bad&evidence_locator[]=bad&evidence_revision[]=1" + ) + + refute has_element?(view, "[id^='selected-evidence-source-']") + + {:ok, other_record} = + Sources.upsert_record(ctx.sources["wikidata"], %{ + external_id: "Q900199", + raw: %{"id" => "Q900199"} + }) + + path = + ~p"/connect?#{%{subject: ctx.work.object_id, evidence_revision: other_record.current_revision.id, evidence_locator: "unrelated"}}" + + {:ok, view, _html} = live(conn, path) + refute has_element?(view, "#selected-evidence-source-#{other_record.current_revision.id}") + end + + test "related mapping labels and assignments beyond catalog page one remain discoverable", + ctx do + love = word!(ctx, "love", ["wordnet"]) + + sense = + sense!(ctx, love, "wordnet", + external_id: "oewn-07558676-n#love", + gloss: "a strong positive emotion of regard and affection" + ) + + target = + Enum.reduce(1..501, nil, fn index, target -> + label = "Scale Artwork #{String.pad_leading(to_string(index), 3, "0")}" + slug = "scale-artwork-#{index}" + {:ok, work} = Registry.create_work(%{preferred_label: label, work_kind: "artwork"}) + + genes = + if index == 501, + do: [%{"id" => "4de292fcef72520001005fe5", "name" => "Love"}], + else: [%{"id" => "irrelevant-gene", "name" => "Irrelevant"}] + + {:ok, record} = + Sources.upsert_record(ctx.sources["artsy"], %{ + external_id: "artwork:scale-#{index}", + url: "https://www.artsy.net/artwork/#{slug}", + raw: %{ + "artwork" => %{ + "id" => "scale-#{index}", + "slug" => slug, + "title" => label, + "category" => "Painting" + }, + "genes" => genes + } + }) + + Repo.insert!(%MaterializedOutput{ + source_record_id: record.id, + output_role: "concept", + output_key: "scale-#{index}", + output_object_id: work.object_id + }) + + if index == 501, do: work, else: target + end) + + assert %{installed: 1} = Artworks.install_meaning_mappings!() + + assert [%{artwork: artwork, sense_id: sense_id, match_type: "related"}] = + Artworks.suggestions([love.object_id]) + + assert artwork.object_id == target.object_id + assert sense_id == sense.object_id + + {:ok, view, _html} = live(ctx.conn, ~p"/define/love") + assert render(view) =~ "Related Artsy gene" + end + test "rejected creator relationship is hidden from cards and search", ctx do revision = Claims.current_revision(ctx.claim.id) {:ok, _review} = Claims.review(revision.id, :rejected) @@ -242,4 +338,111 @@ defmodule DevilsDictionaryWeb.ArtworkLiveTest do assert Artworks.get(ctx.work.object_id).creators == [] refute Enum.any?(Artworks.search("Fixture Painter"), &(&1.object_id == ctx.work.object_id)) end + + test "duplicate source relationships render once while their assertions remain", ctx do + assertion_count = Repo.aggregate(Assertion, :count) + + {:ok, _wikidata_claim} = + Claims.assert(ctx.work.object_id, "authored_by", ctx.creator.object_id, %{ + source_id: ctx.sources["wikidata"].id, + origin_key: "wikidata:duplicate-creator" + }) + + {:ok, _artsy_claim} = + Claims.assert(ctx.work.object_id, "authored_by", ctx.creator.object_id, %{ + source_id: ctx.sources["artsy"].id, + origin_key: "artsy:duplicate-creator" + }) + + assert Repo.aggregate(Assertion, :count) == assertion_count + 2 + assert [%{object_id: creator_id}] = Artworks.get(ctx.work.object_id).creators + assert creator_id == ctx.creator.object_id + + page = EntityPage.build(ctx.creator.object_id) + assert page.pagination.works.count == 1 + assert [%{object_id: work_id}] = page.works + assert work_id == ctx.work.object_id + + {:ok, view, _html} = + live( + ctx.conn, + ~p"/entities/#{ctx.work.object_id}/#{DevilsDictionary.Claims.Connection.slugify(ctx.work.preferred_label)}" + ) + + assert has_element?(view, "#artwork-metadata a[href^='/entities/#{ctx.creator.object_id}/']") + + refute has_element?( + view, + "#entity-connections a[href^='/entities/#{ctx.creator.object_id}/']" + ) + end + + test "a retained provider thumbnail is displayed with the provider's own credit", ctx do + # The independent image projection can be withheld (source-identity gating, + # or a provider-only record) while its credit string survives in metadata. + # Pairing them separately printed a Wikimedia credit under an Artsy image. + Repo.get_by!(Registry.Entity, object_id: ctx.work.object_id) + |> Ecto.Changeset.change( + metadata: %{"image_attribution" => "Some_File.jpg \u00b7 Wikimedia Commons"} + ) + |> Repo.update!() + + artwork = Artworks.get(ctx.work.object_id) + + assert artwork.image_url == "https://images.example.test/fixture.jpg" + assert artwork.image_attribution == "Fixture credit" + refute artwork.image_attribution =~ "Wikimedia" + end + + test "the displayed credit always describes the displayed image", ctx do + provider_thumbnail = "https://images.example.test/fixture.jpg" + + Repo.get_by!(Registry.Entity, object_id: ctx.work.object_id) + |> Ecto.Changeset.change( + metadata: %{ + "image_url" => "https://upload.example.test/independent.jpg", + "image_attribution" => "Independent_File.jpg \u00b7 Wikimedia Commons" + } + ) + |> Repo.update!() + + artwork = Artworks.get(ctx.work.object_id) + + if artwork.image_url == provider_thumbnail do + assert artwork.image_attribution == "Fixture credit" + else + refute artwork.image_attribution == "Fixture credit" + end + end + + test "a definition's artwork candidate reaches the composer only for a contributor", ctx do + war = word!(ctx, "war", ["wordnet"]) + + sense = + sense!(ctx, war, "wordnet", + external_id: "oewn-00975181-n#war", + gloss: "organized armed conflict or warfare" + ) + + assert %{installed: 1} = Artworks.install_meaning_mappings!() + + {:ok, anonymous, _html} = live(ctx.conn, ~p"/define/war") + + assert has_element?( + anonymous, + "#artwork-candidate-#{ctx.work.object_id}-#{sense.object_id}" + ) + + refute has_element?(anonymous, "#artwork-candidates a[href^='/connect?']") + + %{conn: conn, user: user} = register_and_log_in_user(%{conn: ctx.conn}) + Repo.update!(Ecto.Changeset.change(user, internal_contributor: true)) + + {:ok, contributor, _html} = live(conn, ~p"/define/war") + + assert has_element?( + contributor, + "#artwork-candidates a[href^='/connect?'][href*='predicate=illustrates']" + ) + end end From 3a3d55dd9ab58c3be443b0923c2b955b74771091 Mon Sep 17 00:00:00 2001 From: Holden Thomas Date: Sun, 13 Sep 2026 11:47:28 +0200 Subject: [PATCH 6/8] Apply the still-valid CodeRabbit review findings Triaged every open CodeRabbit finding on PR #98 against the current head. Most were already repaired by earlier commits; these four were not. - refuse a checkpoint whose exact identities differ from the manifest being run. `--resume` silently substituted a checkpoint from a different selection, and `--build --resume` made that easy: the base manifest is rediscovered while the stale checkpoint still wins. - treat a live but disabled coordinator as unavailable. The pid clause checked only liveness, so a coordinator that withdrawal had disabled still reported the provider as available. - keep the coordinator authoritative when the database is unreachable. The DBConnection.OwnershipError rescue returned :ok on credentials alone, which reported a withdrawn provider as available. - reject a non-binary `revision` query parameter instead of raising FunctionClauseError on `?revision[]=1`. Also filter backfill QIDs independently rather than per candidate, so a creator who exists locally is hydrated even when that particular artwork is not in the catalog. Titian was skipped 17 times by the old per-candidate filter. Skipped, with reasons: the request-budget, endpoint-validation, redirect-body, creator-search, seeder-identifier, artist-slug, creator-gap, transaction, paged-creator-merge, retry-accounting, discovery-error, manifest-state and test-coverage findings are all already fixed at this head. CodeRabbit's new evidence-provenance finding reads the pre-fix `preselected_evidence/2`; the current arity-3 version binds evidence to the subject's own Artsy record and a regression test already covers the unrelated-revision attack it describes. One existing test pinned `coordinator_unavailable` for an absent coordinator. It passed only because the rescue skipped the coordinator check; the boundary now refuses earlier and reports the same code the database-readable path does. mix precommit: 960 tests, 0 failures with corpus archives present. Co-Authored-By: Claude Opus 5 --- lib/devils_dictionary/artsy/availability.ex | 29 ++++++++-- lib/devils_dictionary/artworks/manifest.ex | 17 ++++++ .../live/connection_live.ex | 6 +- lib/mix/tasks/dd.artworks.seed.ex | 31 +++++++++-- .../artsy/availability_test.exs | 55 +++++++++++++++++++ test/devils_dictionary/artsy/client_test.exs | 6 +- .../artworks/manifest_test.exs | 19 +++++++ .../live/artwork_live_test.exs | 10 ++++ 8 files changed, 160 insertions(+), 13 deletions(-) create mode 100644 test/devils_dictionary/artsy/availability_test.exs diff --git a/lib/devils_dictionary/artsy/availability.ex b/lib/devils_dictionary/artsy/availability.ex index 33368d7c..f27e0abd 100644 --- a/lib/devils_dictionary/artsy/availability.ex +++ b/lib/devils_dictionary/artsy/availability.ex @@ -56,11 +56,20 @@ defmodule DevilsDictionary.Artsy.Availability do true -> :ok end rescue + # Without the database we cannot read source lifecycle, but the coordinator + # is in-memory and still authoritative. Skipping it here would report a + # withdrawn provider as available. DBConnection.OwnershipError -> - if(present?(client_id) and present?(client_secret), - do: :ok, - else: {:error, :credentials_missing} - ) + cond do + not (present?(client_id) and present?(client_secret)) -> + {:error, :credentials_missing} + + not coordinator_enabled?(coordinator) -> + {:error, :source_withdrawn} + + true -> + :ok + end end @doc "Runs a provider-data write while holding the same source lock as withdrawal." @@ -93,14 +102,22 @@ defmodule DevilsDictionary.Artsy.Availability do defp coordinator_enabled?(nil), do: true - defp coordinator_enabled?(coordinator) when is_pid(coordinator), - do: Process.alive?(coordinator) + # Liveness is not permission: a running coordinator that withdrawal has + # disabled must still report the provider as unavailable. + defp coordinator_enabled?(coordinator) when is_pid(coordinator) do + Process.alive?(coordinator) and + DevilsDictionary.Artsy.RequestCoordinator.enabled?(coordinator) + catch + :exit, _ -> false + end defp coordinator_enabled?(coordinator) do case Process.whereis(coordinator) do nil -> false _pid -> DevilsDictionary.Artsy.RequestCoordinator.enabled?(coordinator) end + catch + :exit, _ -> false end defp config, do: Application.get_env(:devils_dictionary, :artsy, []) diff --git a/lib/devils_dictionary/artworks/manifest.ex b/lib/devils_dictionary/artworks/manifest.ex index 616c1da4..6e73da50 100644 --- a/lib/devils_dictionary/artworks/manifest.ex +++ b/lib/devils_dictionary/artworks/manifest.ex @@ -40,6 +40,23 @@ defmodule DevilsDictionary.Artworks.Manifest do upgrade(manifest) end + @doc """ + Whether two manifests describe the same ordered set of exact identities. + + A checkpoint is a manifest whose import state has moved on, so its own + checksum always differs from the base manifest's. Identity is what must not + differ: resuming a checkpoint built from a different selection would silently + process the wrong candidates. + """ + def same_selection?(left, right) when is_map(left) and is_map(right), + do: identities(left) == identities(right) + + defp identities(manifest) do + manifest + |> Map.get("candidates", []) + |> Enum.map(&{&1["qid"], &1["artsy_artwork_slug"]}) + end + @doc "Atomically saves a manifest with a refreshed checksum." def save!(manifest, path) do manifest = manifest |> Map.put("updated_at", timestamp()) |> put_checksum() diff --git a/lib/devils_dictionary_web/live/connection_live.ex b/lib/devils_dictionary_web/live/connection_live.ex index 9e23be35..63cbf101 100644 --- a/lib/devils_dictionary_web/live/connection_live.ex +++ b/lib/devils_dictionary_web/live/connection_live.ex @@ -270,13 +270,17 @@ defmodule DevilsDictionaryWeb.ConnectionLive do defp revision_number(nil), do: {:ok, nil} - defp revision_number(value) do + defp revision_number(value) when is_binary(value) do case Integer.parse(value) do {n, ""} when n > 0 -> {:ok, n} _ -> :error end end + # `?revision[]=1` arrives as a list. A malformed query string is a bad + # request, not a crash. + defp revision_number(_value), do: :error + defp maybe_reject_edit(socket, :edit, false, id) do socket |> put_flash(:error, "Only the original submitter or a reviewer can revise this connection.") diff --git a/lib/mix/tasks/dd.artworks.seed.ex b/lib/mix/tasks/dd.artworks.seed.ex index 037d2506..78a8a7dd 100644 --- a/lib/mix/tasks/dd.artworks.seed.ex +++ b/lib/mix/tasks/dd.artworks.seed.ex @@ -92,7 +92,7 @@ defmodule Mix.Tasks.Dd.Artworks.Seed do manifest = if opts[:resume] && File.exists?(checkpoint_path), - do: Manifest.load!(checkpoint_path), + do: resumed_manifest!(checkpoint_path, base_manifest, path), else: base_manifest wikidata_stats = @@ -166,20 +166,41 @@ defmodule Mix.Tasks.Dd.Artworks.Seed do ) end + # Resuming a checkpoint that belongs to a different manifest would silently + # import the wrong candidate set. `--build --resume` makes that easy: the base + # manifest is rediscovered while a stale checkpoint still wins. + defp resumed_manifest!(checkpoint_path, base_manifest, path) do + checkpoint = Manifest.load!(checkpoint_path) + + unless Manifest.same_selection?(checkpoint, base_manifest) do + Mix.raise( + "checkpoint #{checkpoint_path} does not match the manifest #{path}. " <> + "Its exact identities differ, so resuming it would import a different " <> + "candidate set. Pass --checkpoint for the matching checkpoint, or rerun " <> + "without --resume." + ) + end + + checkpoint + end + defp hydrate_wikidata(manifest, opts, hydrate_opts \\ []) do existing_only? = hydrate_opts[:existing_only] || false + # The rule is "hydrate identities that already exist locally". Applying it + # per candidate instead of per QID dropped creators who do exist locally + # merely because one of their artworks does not. qids = manifest["candidates"] - |> Enum.filter(fn candidate -> - not existing_only? or - not is_nil(DevilsDictionary.Registry.by_external_id("wikidata", candidate["qid"])) - end) |> Enum.flat_map(fn candidate -> [candidate["qid"] | Enum.map(candidate["creators"] || [], & &1["qid"])] end) |> Enum.filter(&is_binary/1) |> Enum.uniq() + |> Enum.filter(fn qid -> + not existing_only? or + not is_nil(DevilsDictionary.Registry.by_external_id("wikidata", qid)) + end) if qids == [] do %{skipped: true, reason: "no_matching_local_identities", requests: 0, records: 0} diff --git a/test/devils_dictionary/artsy/availability_test.exs b/test/devils_dictionary/artsy/availability_test.exs new file mode 100644 index 00000000..cdb6e393 --- /dev/null +++ b/test/devils_dictionary/artsy/availability_test.exs @@ -0,0 +1,55 @@ +defmodule DevilsDictionary.Artsy.AvailabilityTest do + # `async: true` keeps this test's sandbox unshared, which is what lets the + # task below reach the `DBConnection.OwnershipError` path on purpose. + use DevilsDictionary.DataCase, async: true + + alias DevilsDictionary.Artsy.{Availability, RequestCoordinator} + + setup do + {:ok, coordinator} = RequestCoordinator.start_link(name: nil) + %{coordinator: coordinator} + end + + test "a live but disabled coordinator withdraws the provider", ctx do + assert Availability.status_with_credentials("id", "secret", ctx.coordinator) == :ok + + {:ok, _generation} = RequestCoordinator.disable(ctx.coordinator) + + assert Availability.status_with_credentials("id", "secret", ctx.coordinator) == + {:error, :source_withdrawn} + end + + test "withdrawal still wins when the database is unreachable", ctx do + {:ok, _generation} = RequestCoordinator.disable(ctx.coordinator) + + # No sandbox ownership in this process, so reading source lifecycle raises + # DBConnection.OwnershipError and the rescue clause decides. The coordinator + # is in memory and remains authoritative, so the answer must not be `:ok`. + result = + Task.async(fn -> + Availability.status_with_credentials("id", "secret", ctx.coordinator) + end) + |> Task.await() + + assert result == {:error, :source_withdrawn} + end + + test "missing credentials are still reported when the database is unreachable", ctx do + result = + Task.async(fn -> + Availability.status_with_credentials("", "", ctx.coordinator) + end) + |> Task.await() + + assert result == {:error, :credentials_missing} + end + + test "a dead coordinator fails closed", ctx do + Process.unlink(ctx.coordinator) + Process.exit(ctx.coordinator, :kill) + refute Process.alive?(ctx.coordinator) + + assert Availability.status_with_credentials("id", "secret", ctx.coordinator) == + {:error, :source_withdrawn} + end +end diff --git a/test/devils_dictionary/artsy/client_test.exs b/test/devils_dictionary/artsy/client_test.exs index 65f45813..6960cf1a 100644 --- a/test/devils_dictionary/artsy/client_test.exs +++ b/test/devils_dictionary/artsy/client_test.exs @@ -244,7 +244,11 @@ defmodule DevilsDictionary.Artsy.ClientTest do {client, calls} = client([response(201, %{"token" => "unused"})], coordinator: :missing_artsy_coordinator) - assert {:error, %{code: "coordinator_unavailable"}, client} = + # An absent coordinator is refused at the availability boundary, which is + # earlier than the reservation. `Availability` reports an unreachable + # coordinator as a withdrawn source whether or not the database is + # readable; what matters here is that nothing leaves the server. + assert {:error, %{code: "source_withdrawn"}, client} = Client.artwork(client, "work-one") assert client.request_count == 0 diff --git a/test/devils_dictionary/artworks/manifest_test.exs b/test/devils_dictionary/artworks/manifest_test.exs index 463fe496..03699e4c 100644 --- a/test/devils_dictionary/artworks/manifest_test.exs +++ b/test/devils_dictionary/artworks/manifest_test.exs @@ -18,6 +18,25 @@ defmodule DevilsDictionary.Artworks.ManifestTest do assert_raise ArgumentError, ~r/checksum mismatch/, fn -> Manifest.load!(path) end end + test "a checkpoint matches its own manifest and never a different selection" do + one = %{"qid" => "Q1", "artsy_artwork_slug" => "one", "kind" => "painting"} + two = %{"qid" => "Q2", "artsy_artwork_slug" => "two", "kind" => "painting"} + + base = Manifest.new([one, two]) + + # A checkpoint is the same selection with import state moved on. Its own + # checksum differs, but its identities must not. + checkpoint = Manifest.update_candidate(base, 0, %{"status" => "matched"}) + refute checkpoint["checksum"] == base["checksum"] + assert Manifest.same_selection?(checkpoint, base) + + refute Manifest.same_selection?(Manifest.new([one]), base) + refute Manifest.same_selection?(Manifest.new([two, one]), base) + + renamed = %{"qid" => "Q2", "artsy_artwork_slug" => "two-b", "kind" => "painting"} + refute Manifest.same_selection?(Manifest.new([one, renamed]), base) + end + test "completed statuses are resumable while quota is retried" do for status <- ~w(matched created skipped conflict unavailable) do assert Manifest.completed?(%{"import" => %{"status" => status}}) diff --git a/test/devils_dictionary_web/live/artwork_live_test.exs b/test/devils_dictionary_web/live/artwork_live_test.exs index f56cc860..bca870ba 100644 --- a/test/devils_dictionary_web/live/artwork_live_test.exs +++ b/test/devils_dictionary_web/live/artwork_live_test.exs @@ -445,4 +445,14 @@ defmodule DevilsDictionaryWeb.ArtworkLiveTest do "#artwork-candidates a[href^='/connect?'][href*='predicate=illustrates']" ) end + + test "a malformed revision query parameter is rejected, not a crash", ctx do + # `?revision[]=1` arrives as a list rather than a binary. + {:ok, malformed, html} = live(ctx.conn, "/connections/#{ctx.claim.id}?revision[]=1") + assert html =~ "no such connection" + refute render(malformed) =~ "revision 1" + + {:ok, view, _html} = live(ctx.conn, "/connections/#{ctx.claim.id}?revision=1") + assert render(view) =~ "revision 1" + end end From 7d21c78c54cebf4faa40fc7f6c2eae7e77c22d3a Mon Sep 17 00:00:00 2001 From: Holden Thomas Date: Sun, 13 Sep 2026 12:08:59 +0200 Subject: [PATCH 7/8] Redact credentials from client inspection; correct stale budget wording Issue 86 requires no raw credential or token logs. The Artsy client struct carries both and had no Inspect derivation, so a crash report or a stray inspect/1 could print them. Derive Inspect excluding client_secret and token, with a test that asserts neither value appears. The feasibility report still described the interactive allowance as a single server-boot budget; it renews per configured window since the second re-audit repair. Co-Authored-By: Claude Fable 5.1 --- .../integrations/artsy-feasibility-2026-09-12.md | 2 +- lib/devils_dictionary/artsy/client.ex | 3 +++ test/devils_dictionary/artsy/client_test.exs | 16 ++++++++++++++++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/docs/integrations/artsy-feasibility-2026-09-12.md b/docs/integrations/artsy-feasibility-2026-09-12.md index 9cd5e93b..03d99364 100644 --- a/docs/integrations/artsy-feasibility-2026-09-12.md +++ b/docs/integrations/artsy-feasibility-2026-09-12.md @@ -57,7 +57,7 @@ title search. Widen only after the pilot reports its actual coverage. | P2042 artist crosswalk | Wikidata P2042 is an artist slug | `vincent-van-gogh` redirected, then returned opaque ID `4d8b92944eb68a1b2c000264` | Resolve creators through exact slug/ID evidence, never through a name string. | | Direct gene assignments | Artwork links expose assigned genes | Goya returned 5 assignments: 19th Century, Chiaroscuro, Collective History, Conflict, Cultural Commentary | Preserve as attributed Artsy vocabulary. No string-to-meaning equivalence. | | Gene-filtered acquisition | Gene resources link to artworks | The first 10 `gene_id=19th Century` artwork IDs were identical to the unfiltered first 10 | **Unverified/unsafe.** Do not seed through gene traversal. A direct per-work assignment may label a review candidate only. | -| Retry/quota signal | 5 requests/sec documented | Probe throttled below 3/sec; 0 retries, 0 quota responses | One application-wide coordinator paces all callers. It honors the full `Retry-After`, bounds retries and total attempts, and reports local request exhaustion separately from provider quota exhaustion. Interactive discovery has one atomic 30-attempt server-boot allowance that browser reloads and concurrent LiveViews cannot reset. | +| Retry/quota signal | 5 requests/sec documented | Probe throttled below 3/sec; 0 retries, 0 quota responses | One application-wide coordinator paces all callers. It honors the full `Retry-After`, bounds retries and total attempts, and reports local request exhaustion separately from provider quota exhaustion. Interactive discovery shares one atomic server-side allowance, 30 attempts per 60-second window by default, that browser reloads and concurrent LiveViews cannot bypass; it renews on the window rather than only on restart. | ## Repair feasibility findings diff --git a/lib/devils_dictionary/artsy/client.ex b/lib/devils_dictionary/artsy/client.ex index 804f85da..5ce8a8db 100644 --- a/lib/devils_dictionary/artsy/client.ex +++ b/lib/devils_dictionary/artsy/client.ex @@ -16,6 +16,9 @@ defmodule DevilsDictionary.Artsy.Client do @default_retries 2 @default_rate_limit_ms 340 + # A crash report or a stray `inspect/1` must never print the credential or + # the bearer token this struct carries. + @derive {Inspect, except: [:client_secret, :token]} defstruct base_url: @base_url, client_id: nil, client_secret: nil, diff --git a/test/devils_dictionary/artsy/client_test.exs b/test/devils_dictionary/artsy/client_test.exs index 6960cf1a..51282697 100644 --- a/test/devils_dictionary/artsy/client_test.exs +++ b/test/devils_dictionary/artsy/client_test.exs @@ -355,4 +355,20 @@ defmodule DevilsDictionary.Artsy.ClientTest do end defp response(status, body), do: %Req.Response{status: status, body: body} + + test "inspecting a client never prints the credential or the token" do + client = + Client.new( + client_id: "fixture-id", + client_secret: "fixture-secret-value", + coordinator: nil + ) + + client = %{client | token: "fixture-xapp-token-value"} + printed = inspect(client) + + assert printed =~ "fixture-id" + refute printed =~ "fixture-secret-value" + refute printed =~ "fixture-xapp-token-value" + end end From 2debb3a705e72791256683a29c23c5ed636d023f Mon Sep 17 00:00:00 2001 From: Holden Thomas Date: Sun, 13 Sep 2026 12:13:41 +0200 Subject: [PATCH 8/8] Cover the remaining tests issue 86 names explicitly A close-out audit of every test the issue and its readiness audit require found the implementation present but these behaviours asserted only implicitly or not at all: - provider-off local operation: catalog notice, disabled live check, local search and word-page candidates with no credentials configured - search hits of an unsupported type or on a foreign or downgraded endpoint are labeled and never fetched - safe_api_url?/2 rejects http, non-default ports, paths outside /api/, userinfo and other hosts - a response arriving after withdrawal is discarded rather than published (mutation-checked: removing the post-response generation guard fails it) - a renamed provider slug on a known opaque ID opens reconciliation instead of rebinding or duplicating - a candidate without Wikidata identity is never seeded and never merged into a same-title local work - a hydrated artwork with no genes completes and yields no candidate vocabulary - an artwork with no image renders the placeholder on the card and the page mix precommit: 969 tests, 0 failures with corpus archives present. Co-Authored-By: Claude Fable 5.1 --- test/devils_dictionary/artsy/client_test.exs | 80 +++++++++++++++++++ .../artworks/seeder_test.exs | 63 +++++++++++++++ .../live/artwork_live_test.exs | 44 ++++++++++ 3 files changed, 187 insertions(+) diff --git a/test/devils_dictionary/artsy/client_test.exs b/test/devils_dictionary/artsy/client_test.exs index 51282697..d6280453 100644 --- a/test/devils_dictionary/artsy/client_test.exs +++ b/test/devils_dictionary/artsy/client_test.exs @@ -371,4 +371,84 @@ defmodule DevilsDictionary.Artsy.ClientTest do refute printed =~ "fixture-secret-value" refute printed =~ "fixture-xapp-token-value" end + + test "search hits of another type or on a foreign endpoint are labeled, never fetched" do + responses = [ + response(201, %{"token" => "fixture-token"}), + response(200, %{ + "_embedded" => %{ + "results" => [ + %{ + "type" => "artist", + "title" => "Some Artist", + "_links" => %{ + "self" => %{"href" => "https://api.artsy.net/api/artists/some-artist"} + } + }, + %{ + "type" => "artwork", + "title" => "Elsewhere", + "_links" => %{"self" => %{"href" => "https://evil.example.test/api/artworks/x"}} + }, + %{ + "type" => "artwork", + "title" => "Downgraded", + "_links" => %{"self" => %{"href" => "http://api.artsy.net/api/artworks/y"}} + } + ] + }, + "_links" => %{} + }) + ] + + {client, calls} = client(responses) + assert {:ok, result, _client} = Client.search_artworks(client, "war", size: 3) + + assert Enum.map(result.items, & &1.status) == + [:unsupported_type, :invalid_endpoint, :invalid_endpoint] + + # Token and search only: none of the three hits was fetched. + assert length(Agent.get(calls, & &1)) == 2 + end + + test "only https URLs on the configured host under /api/ without userinfo are safe" do + client = Client.new(client_id: "id", client_secret: "secret", coordinator: nil) + + assert Client.safe_api_url?(client, "https://api.artsy.net/api/artworks/x") + refute Client.safe_api_url?(client, "http://api.artsy.net/api/artworks/x") + refute Client.safe_api_url?(client, "https://api.artsy.net:8443/api/artworks/x") + refute Client.safe_api_url?(client, "https://api.artsy.net/artworks/x") + refute Client.safe_api_url?(client, "https://user:pw@api.artsy.net/api/artworks/x") + refute Client.safe_api_url?(client, "https://www.artsy.net/api/artworks/x") + refute Client.safe_api_url?(client, nil) + end + + test "a response that arrives after withdrawal is discarded, never published" do + coordinator = + start_supervised!({RequestCoordinator, name: nil, interval_ms: 0}, id: make_ref()) + + queue = + start_supervised!( + {Agent, + fn -> [response(201, %{"token" => "fixture-token"}), artwork_response("work-one")] end}, + id: make_ref() + ) + + transmitted = start_supervised!({Agent, fn -> 0 end}, id: make_ref()) + + request_fun = fn options -> + Agent.update(transmitted, &(&1 + 1)) + reply = Agent.get_and_update(queue, fn [r | rest] -> {{:ok, r}, rest} end) + # Withdrawal lands while the artwork request is in flight. + if String.contains?(options[:url], "/api/artworks/"), + do: RequestCoordinator.disable(coordinator) + + reply + end + + {client, _calls} = client([], coordinator: coordinator, request_fun: request_fun) + + assert {:error, %{code: "provider_disabled"}, _client} = Client.artwork(client, "work-one") + assert Agent.get(transmitted, & &1) == 2 + end end diff --git a/test/devils_dictionary/artworks/seeder_test.exs b/test/devils_dictionary/artworks/seeder_test.exs index edd8a8c1..3be544c9 100644 --- a/test/devils_dictionary/artworks/seeder_test.exs +++ b/test/devils_dictionary/artworks/seeder_test.exs @@ -519,4 +519,67 @@ defmodule DevilsDictionary.Artworks.SeederTest do "mainsnak" => %{"datavalue" => %{"value" => value, "type" => "string"}} } end + + test "a renamed provider slug on a known opaque ID reconciles instead of rebinding" do + {:ok, work} = Registry.create_work(%{preferred_label: "Fixture Work", work_kind: "artwork"}) + {:ok, _} = Registry.add_external_id(work.object_id, "wikidata", "Q900086") + {:ok, _} = Registry.add_external_id(work.object_id, "artsy_artwork_slug", "fixture-work") + {:ok, _} = Registry.add_external_id(work.object_id, "artsy_artwork_id", "opaque-work-86") + + responses = [ + token(), + response(200, artwork("opaque-work-86", "fixture-work-renamed", "Painting")) + ] + + {_manifest, summary} = + Seeder.run(Manifest.new([candidate()]), artsy_client: client(responses), record_limit: 1) + + assert summary.conflicts == 1 + assert summary.created == 0 + assert Registry.by_external_id("artsy_artwork_id", "opaque-work-86") == work.object_id + assert Registry.by_external_id("artsy_artwork_slug", "fixture-work") == work.object_id + assert Registry.by_external_id("artsy_artwork_slug", "fixture-work-renamed") == nil + assert Repo.aggregate(ReconciliationCase, :count) == 1 + end + + test "a candidate without Wikidata identity is never seeded and never title-merged" do + {:ok, same_title} = + Registry.create_work(%{preferred_label: "Fixture Work", work_kind: "artwork"}) + + orphan = Map.delete(candidate(), "qid") + + {_manifest, summary} = + Seeder.run(Manifest.new([orphan]), + artsy_client: client(painting_responses()), + record_limit: 1 + ) + + assert summary.requests == 0 + assert summary.created == 0 + assert summary.matched == 0 + assert Registry.by_external_id("artsy_artwork_slug", "fixture-work") == nil + assert Registry.by_external_id("artsy_artwork_id", "opaque-work-86") == nil + assert Registry.by_external_id("artsy_artwork_slug", "fixture-work") != same_title.object_id + end + + test "a hydrated artwork with no genes completes and carries no candidate vocabulary" do + responses = [ + token(), + response(200, artwork("opaque-work-86", "fixture-work", "Painting")), + response(200, %{"_embedded" => %{"artists" => []}}), + response(200, %{"_embedded" => %{"genes" => []}}) + ] + + {manifest, summary} = + Seeder.run(Manifest.new([candidate()]), artsy_client: client(responses), record_limit: 1) + + assert summary.unavailable == 0 + assert summary.created + summary.matched == 1 + + assert get_in(manifest, ["candidates", Access.at(0), "import", "stages", "genes", "status"]) == + "complete" + + work_id = Registry.by_external_id("wikidata", "Q900086") + assert Artworks.get(work_id).genes == [] + end end diff --git a/test/devils_dictionary_web/live/artwork_live_test.exs b/test/devils_dictionary_web/live/artwork_live_test.exs index bca870ba..7227c507 100644 --- a/test/devils_dictionary_web/live/artwork_live_test.exs +++ b/test/devils_dictionary_web/live/artwork_live_test.exs @@ -455,4 +455,48 @@ defmodule DevilsDictionaryWeb.ArtworkLiveTest do {:ok, view, _html} = live(ctx.conn, "/connections/#{ctx.claim.id}?revision=1") assert render(view) =~ "revision 1" end + + test "with the provider unconfigured the catalog, search and word page work locally", ctx do + refute DevilsDictionary.Artsy.Availability.configured?() + + {:ok, view, _html} = live(ctx.conn, ~p"/artworks") + assert has_element?(view, "#artsy-disabled") + assert has_element?(view, "#artsy-search-submit[disabled]") + + html = view |> form("#artwork-search", search: %{q: "Fixture Painter"}) |> render_change() + assert html =~ ~s(id="artwork-#{ctx.work.object_id}") + + war = word!(ctx, "war", ["wordnet"]) + sense!(ctx, war, "wordnet", external_id: "oewn-00975181-n#war", gloss: "armed conflict") + assert %{installed: 1} = Artworks.install_meaning_mappings!() + + {:ok, word_view, _html} = live(ctx.conn, ~p"/define/war") + assert has_element?(word_view, "#artwork-candidates") + end + + test "an artwork with no image renders an honest placeholder, not a broken entry", ctx do + {:ok, bare} = + Registry.create_work(%{preferred_label: "Bare Fixture", work_kind: "artwork"}) + + {:ok, view, _html} = live(ctx.conn, ~p"/artworks") + + assert has_element?(view, "#artwork-#{bare.object_id}-image[data-image-state='empty']") + + assert has_element?( + view, + "#artwork-#{bare.object_id}-image [data-artwork-fallback]:not([hidden])" + ) + + refute has_element?(view, "#artwork-#{bare.object_id}-image img[data-artwork-image]") + assert render(view) =~ "Bare Fixture" + + {:ok, page, _html} = + live( + ctx.conn, + ~p"/entities/#{bare.object_id}/#{DevilsDictionary.Claims.Connection.slugify("Bare Fixture")}" + ) + + assert has_element?(page, "#entity-artwork[data-image-state='empty']") + assert has_element?(page, "#entity-artwork [data-artwork-fallback]:not([hidden])") + end end