Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -4,5 +4,9 @@ CINEGRAPH_GRAPHQL_URL=https://cinegraph.org/api/graphql
GIPHY_API_KEY=
DISCOVERY_POSITIVE_REFRESH_SECONDS=2592000
DISCOVERY_EMPTY_REFRESH_SECONDS=86400
# Optional per-source freshness overrides, for example seven days:
# Optional per-source freshness overrides, for example seven days. The accepted
# names are derived from the provider registry, so every registered provider
# gets <SLUG>_DISCOVERY_POSITIVE_REFRESH_SECONDS and
# <SLUG>_DISCOVERY_EMPTY_REFRESH_SECONDS without another list to edit.
CINEGRAPH_DISCOVERY_EMPTY_REFRESH_SECONDS=
ARTSY_DISCOVERY_EMPTY_REFRESH_SECONDS=
9 changes: 9 additions & 0 deletions config/config.exs
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,15 @@ config :devils_dictionary,
:user_agent,
"wordhoard/0.1 (https://github.com/razrfly/dictionary; holden.thomas@gmail.com)"

# The registry. Adding a provider is this list plus its module: the source
# catalog, `mix dd.discovery`, the freshness overrides in runtime.exs and the
# page all read it, and none of them names a provider itself.
config :devils_dictionary, :discovery_providers, [
DevilsDictionary.Discovery.Providers.Artsy,
DevilsDictionary.Discovery.Providers.CineGraph,
DevilsDictionary.Discovery.Providers.Giphy
]

# Culture discovery is visit-driven and bounded. Provider modules own their
# capability differences; these are operating limits for the shared lifecycle.
config :devils_dictionary, :discovery,
Expand Down
76 changes: 49 additions & 27 deletions config/runtime.exs
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,34 @@ config :devils_dictionary, DevilsDictionaryWeb.Endpoint,
# Load provider settings from a local .env only in development.
# Exported environment variables take precedence. Values are literal: no shell
# expansion or evaluation, and secrets never appear in parsing errors.
# The per-provider policy variables are derived from the same registry the
# override loop below reads, so a provider added to the registry cannot have its
# .env overrides silently discarded by an allowlist nobody remembered to edit.
discovery_provider_policy_env =
:devils_dictionary
|> Application.get_env(:discovery_providers, [])
|> Enum.flat_map(fn provider ->
prefix = String.upcase(provider.slug())

[
"#{prefix}_DISCOVERY_POSITIVE_REFRESH_SECONDS",
"#{prefix}_DISCOVERY_EMPTY_REFRESH_SECONDS"
]
end)

# Credentials stay an explicit list: each one is a deliberate decision about a
# secret, and no naming rule should be able to widen it.
allowed_provider_env =
[
"CINEGRAPH_API_KEY",
"CINEGRAPH_GRAPHQL_URL",
"GIPHY_API_KEY",
"ARTSY_CLIENT_ID",
"ARTSY_CLIENT_SECRET",
"DISCOVERY_POSITIVE_REFRESH_SECONDS",
"DISCOVERY_EMPTY_REFRESH_SECONDS"
] ++ discovery_provider_policy_env

local_provider_env =
if config_env() == :dev do
case File.read(Path.expand("../.env", __DIR__)) do
Expand All @@ -37,32 +65,23 @@ local_provider_env =
|> String.split("\n")
|> Enum.reduce(%{}, fn line, values ->
case String.split(String.trim(line), "=", parts: 2) do
[name, value]
when name in [
"CINEGRAPH_API_KEY",
"CINEGRAPH_GRAPHQL_URL",
"GIPHY_API_KEY",
"ARTSY_CLIENT_ID",
"ARTSY_CLIENT_SECRET",
"DISCOVERY_POSITIVE_REFRESH_SECONDS",
"DISCOVERY_EMPTY_REFRESH_SECONDS",
"CINEGRAPH_DISCOVERY_POSITIVE_REFRESH_SECONDS",
"CINEGRAPH_DISCOVERY_EMPTY_REFRESH_SECONDS",
"GIPHY_DISCOVERY_POSITIVE_REFRESH_SECONDS",
"GIPHY_DISCOVERY_EMPTY_REFRESH_SECONDS"
] ->
value = String.trim(value)

value =
if String.length(value) >= 2 and
((String.starts_with?(value, "\"") and String.ends_with?(value, "\"")) or
(String.starts_with?(value, "'") and String.ends_with?(value, "'"))) do
String.slice(value, 1, String.length(value) - 2)
else
value
end

Map.put(values, name, value)
[name, value] when name != "" ->
if name in allowed_provider_env do
value = String.trim(value)

value =
if String.length(value) >= 2 and
((String.starts_with?(value, "\"") and String.ends_with?(value, "\"")) or
(String.starts_with?(value, "'") and String.ends_with?(value, "'"))) do
String.slice(value, 1, String.length(value) - 2)
else
value
end

Map.put(values, name, value)
else
values
end

_ ->
values
Expand Down Expand Up @@ -133,7 +152,10 @@ discovery_config =
source_policies = Keyword.fetch!(discovery_config, :source_policies)

source_policies =
Enum.reduce(["cinegraph", "giphy"], source_policies, fn slug, policies ->
:devils_dictionary
|> Application.get_env(:discovery_providers, [])
|> Enum.map(& &1.slug())
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
|> Enum.reduce(source_policies, fn slug, policies ->
prefix = String.upcase(slug)

overrides =
Expand Down
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
25 changes: 20 additions & 5 deletions lib/devils_dictionary/discovery.ex
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ defmodule DevilsDictionary.Discovery do
end

@doc "Returns the reader state for an enabled provider without making external requests."
def state(target_id, provider_slug \\ "cinegraph") do
def state(target_id, provider_slug) do
with {:ok, provider} <- provider(provider_slug),
%Mapping{} = mapping <- enabled_mapping(target_id, provider.slug()),
{:ok, ^provider, _source} <- eligible_provider_for_mapping(mapping),
Expand Down Expand Up @@ -208,7 +208,7 @@ defmodule DevilsDictionary.Discovery do
{:error, "adapter_version_changed"},
:ok <- provider.validate_mapping(run.mapping.operation, run.mapping.parameters) do
request_fun = fn stage, payload ->
DevilsDictionary.Discovery.Transport.graphql(provider, run.id, stage, payload)
DevilsDictionary.Discovery.Transport.request(provider, run.id, stage, payload)
end

response =
Expand Down Expand Up @@ -1119,6 +1119,11 @@ defmodule DevilsDictionary.Discovery do
end
end

# The capability map is a claim and the exported callbacks are the proof, so
# this gate has to ask for both. `Providers.server_providers/1` already does,
# but `request/3` resolves from `Providers.all/0` and arrives here instead —
# without this clause a module declaring the pipeline and exporting none of it
# would get a mapping and a queued run, and raise where nothing can recover.
defp provider_eligible(provider, %Source{} = source) do
capabilities = provider.capabilities()

Expand All @@ -1127,6 +1132,7 @@ defmodule DevilsDictionary.Discovery do
!provider.enabled?() -> {:error, :provider_disabled}
!capabilities.background -> {:error, :provider_not_background}
capabilities.transport != :server -> {:error, :provider_not_server}
!Providers.retrievable?(provider) -> {:error, :provider_not_retrievable}
true -> :ok
end
end
Expand Down Expand Up @@ -1203,17 +1209,26 @@ defmodule DevilsDictionary.Discovery do
end

defp provider_metadata(provider, source) do
attrs = provider.source_attrs()
content_types = provider.capabilities().content_types

%{
provider_name: if(source, do: source.name, else: provider.source_attrs().name),
provider_attribution:
if(source, do: source.attribution, else: provider.source_attrs().attribution),
provider_name: if(source, do: source.name, else: attrs.name),
provider_attribution: if(source, do: source.attribution, else: attrs.attribution),
# The one-line qualifier a provider wants beside its name on the shelf
# ("keywords: TMDb"). The reader renders whatever is here and knows no
# provider by name.
provider_detail: shelf_detail(provider),
content_types: content_types,
pagination: provider.capabilities().pagination
}
end

defp shelf_detail(provider) do
if Code.ensure_loaded?(provider) and function_exported?(provider, :shelf_detail, 0),
do: provider.shelf_detail()
end

defp in_flight(mapping_id, request_key) do
Repo.one(
from r in Run,
Expand Down
74 changes: 74 additions & 0 deletions lib/devils_dictionary/discovery/content_types.ex
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
defmodule DevilsDictionary.Discovery.ContentTypes do
@moduledoc """
What a content type looks like on a shelf. Data, not code.

A provider declares `content_types` in its capabilities; the reader asks this
table how to present them. Adding a content type is an entry here, not a new
branch in `DevilsDictionaryWeb.Culture` or `WordLive`, which is the whole
point: the shared code stops knowing which providers exist.

`thumbnail_keys` is the ordered ladder read from a result's preview metadata.
A type with no `aspect` has no image slot at all, so a text result renders as
a title and its source rather than an empty poster frame.
"""

@table %{
film: %{
heading: "Films",
label: "film",
badge: "Film",
aspect: "aspect-[2/3]",
icon: "hero-film",
thumbnail_keys: ~w(poster_url still_url image_url media_url)
},
artwork: %{
heading: "Artworks",
label: "artwork",
badge: "Artwork",
aspect: "aspect-square",
icon: "hero-photo",
thumbnail_keys: ~w(image_url thumbnail_url)
},
gif: %{
heading: "GIFs",
label: "GIFs",
badge: nil,
aspect: "aspect-square",
icon: "hero-photo",
thumbnail_keys: ~w(media_url image_url)
},
text: %{
heading: "Texts",
label: "text",
badge: "Text",
aspect: nil,
icon: "hero-document-text",
thumbnail_keys: []
}
}

@known [:film, :artwork, :gif, :text]

@doc "Every content type the reader can present, in shelf order."
def known, do: @known

@doc "True when at least one of `types` can be presented."
def any_known?(types), do: Enum.any?(types, &(&1 in @known))

@doc "The presentation entry for a known content type."
def fetch!(type) when is_map_key(@table, type), do: Map.fetch!(@table, type)

@doc "The presentation entry, or the entry for `default` when the type is unknown."
def get(type, default \\ :film)

def get(type, _default) when is_map_key(@table, type), do: Map.fetch!(@table, type)
def get(_type, default), do: fetch!(default)

@doc "Reads a preview thumbnail using this content type's key ladder."
def thumbnail_url(type, metadata) when is_map(metadata) do
type
|> get()
|> Map.fetch!(:thumbnail_keys)
|> Enum.find_value(&metadata[&1])
end
end
30 changes: 29 additions & 1 deletion lib/devils_dictionary/discovery/provider.ex
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,36 @@ defmodule DevilsDictionary.Discovery.Provider do
| {:error, String.t()}
| {:deferred, String.t(), pos_integer(), map()}

@doc """
Whether an HTTP status is worth another bounded attempt.

The default, applied when a provider does not define this, is `429` or any
`5xx` — a throttle or a server fault. A provider overrides it when its API
reports backpressure some other way, and must keep the default cases
retryable; `Discovery.Transport.default_retryable_status?/1` is public so an
implementation can delegate to it rather than restate it.

This exists because a status code is a provider's dialect, not a fact. The Met
answers a keyless, throttled request with `403` and no `Retry-After`, so for
that provider a `403` is a backoff signal and treating it as an
authentication verdict is both wrong and unrecoverable.
"""
@callback retryable_status?(non_neg_integer()) :: boolean()

@doc """
One short qualifier shown beside the provider name on a shelf, or `nil`.

It exists so the reader can say "CineGraph · keywords: TMDb" without any
component knowing that CineGraph exists. It is not persisted: `source_attrs/0`
is upserted into `sources` column by column, so a key that is not a column
breaks the catalog seed.
"""
@callback shelf_detail() :: String.t() | nil

@optional_callbacks automatic_mapping: 1,
request_options: 1,
validate_mapping: 2,
retrieve: 4
retrieve: 4,
shelf_detail: 0,
retryable_status?: 1
end
50 changes: 43 additions & 7 deletions lib/devils_dictionary/discovery/providers.ex
Original file line number Diff line number Diff line change
@@ -1,13 +1,16 @@
defmodule DevilsDictionary.Discovery.Providers do
@moduledoc "The configured provider registry used by pages, workers and `mix dd.discovery`."
@moduledoc """
The configured provider registry used by pages, workers and `mix dd.discovery`.

alias DevilsDictionary.Discovery.Providers.{Artsy, CineGraph, Giphy}
The list lives in `config :devils_dictionary, :discovery_providers`, so adding
a provider is one config line and nothing else. Registration is deliberately
weaker than retrieval: a module may stay in the registry — and therefore in
the source catalog — while its transport or legal prerequisites are
unresolved. `server_providers/1` is the gate that separates the two, and it
admits only modules that can actually be driven through the pipeline.
"""

@default [Artsy, CineGraph, Giphy]

def all do
Application.get_env(:devils_dictionary, :discovery_providers, @default)
end
def all, do: Application.fetch_env!(:devils_dictionary, :discovery_providers)

def get(slug) when is_atom(slug), do: get(Atom.to_string(slug))

Expand All @@ -25,19 +28,52 @@ defmodule DevilsDictionary.Discovery.Providers do
end)
end

@doc """
Providers the background pipeline can actually run.

A declaration of `transport: :server, background: true` is a claim; exporting
the retrieval callbacks is the proof. Without them the run worker has nothing
to call, so such a module is registered but never scheduled.
"""
def server_providers(content_type \\ nil) do
Enum.filter(all(), fn provider ->
capabilities = provider.capabilities()

provider.enabled?() and capabilities.background and capabilities.transport == :server and
retrievable?(provider) and
(is_nil(content_type) or content_type in capabilities.content_types)
end)
end

@pipeline_callbacks [retrieve: 4, automatic_mapping: 1, request_options: 1]

@doc """
True when the module exports every callback the pipeline drives it through.

`retrieve/4` alone is not enough: `Discovery` builds the automatic mapping
with `automatic_mapping/1` and `Transport` builds the request with
`request_options/1`, so a module exporting one of the three and not the others
would pass the gate and then raise mid-run. All three, or none.
"""
def retrievable?(provider) do
Code.ensure_loaded?(provider) and
Enum.all?(@pipeline_callbacks, fn {callback, arity} ->
function_exported?(provider, callback, arity)
end)
end

@doc "The callbacks `retrievable?/1` requires, as `{name, arity}` pairs."
def pipeline_callbacks, do: @pipeline_callbacks

def supports?(slug, content_type) do
case get(slug) do
nil -> false
provider -> content_type in provider.capabilities().content_types
end
end

@doc "True when the provider declares any of `content_types`."
def supports_any?(slug, content_types) do
Enum.any?(content_types, &supports?(slug, &1))
end
end
3 changes: 3 additions & 0 deletions lib/devils_dictionary/discovery/providers/cine_graph.ex
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,9 @@ defmodule DevilsDictionary.Discovery.Providers.CineGraph do
}
end

@impl true
def shelf_detail, do: "keywords: TMDb"

@impl DevilsDictionary.SourceIdentity.Adapter
def identity_record(
%{
Expand Down
Loading