From 3a2ea18f87293344e1435e1931355064b728d36d Mon Sep 17 00:00:00 2001 From: Pete Birkinshaw Date: Mon, 5 Aug 2024 14:58:35 +0100 Subject: [PATCH] Top-level Publish tests --- lib/smee/publish.ex | 68 ++----- lib/smee/publish/frontend_utils.ex | 42 ++++ .../smee/smee_publish_frontend_utils_test.exs | 81 ++++++++ test/smee/smee_publish_test.exs | 183 ++++++++++++++++-- 4 files changed, 307 insertions(+), 67 deletions(-) create mode 100644 lib/smee/publish/frontend_utils.ex create mode 100644 test/smee/smee_publish_frontend_utils_test.exs diff --git a/lib/smee/publish.ex b/lib/smee/publish.ex index a86082d..584b662 100644 --- a/lib/smee/publish.ex +++ b/lib/smee/publish.ex @@ -76,6 +76,7 @@ defmodule Smee.Publish do alias Smee.Publish.Markdown alias Smee.Publish.Csv alias Smee.Publish.SamlXml + alias Smee.Publish.FrontendUtils use Smee.Publish.LegacyCompatibility @@ -85,21 +86,9 @@ defmodule Smee.Publish do """ @spec formats() :: list(atom()) def formats() do - [ - :csv, - :disco, - :index, - :markdown, - :saml, - :thiss, - :udest, - :udisco - ] + FrontendUtils.formats() end - @default_options [format: :saml, lang: "en", id_type: :hash, to: "published", labels: false] - @allowed_options Keyword.keys(@default_options) ++ [:valid_until, :filename] - @doc """ Estimates the size (in bytes) of an aggregated published file or stream in the selected format (defaulting to SAML2 metadata). @@ -112,8 +101,8 @@ defmodule Smee.Publish do """ @spec eslength(entities :: Enumerable.t(), options :: keyword()) :: integer() def eslength(entities, options \\ []) do - options = prepare_options(options) - apply(select_backend(options), :eslength, [entities, options]) + options = FrontendUtils.prepare_options(options) + apply(FrontendUtils.select_backend(options), :eslength, [entities, options]) end @doc """ @@ -127,8 +116,8 @@ defmodule Smee.Publish do """ @spec aggregate(entities :: Enumerable.t(), options :: keyword()) :: binary() def aggregate(entities, options \\ []) do - options = prepare_options(options) - apply(select_backend(options), :aggregate, [entities, options]) + options = FrontendUtils.prepare_options(options) + apply(FrontendUtils.select_backend(options), :aggregate, [entities, options]) end @doc """ @@ -143,8 +132,8 @@ defmodule Smee.Publish do """ @spec aggregate_stream(entities :: Enumerable.t(), options :: keyword()) :: Enumerable.t(binary()) def aggregate_stream(entities, options \\ []) do - options = prepare_options(options) - apply(select_backend(options), :aggregate_stream, [entities, options]) + options = FrontendUtils.prepare_options(options) + apply(FrontendUtils.select_backend(options), :aggregate_stream, [entities, options]) end @doc """ @@ -159,8 +148,8 @@ defmodule Smee.Publish do """ @spec items(entities :: Enumerable.t(), options :: keyword()) :: Map.t(tuple()) def items(entities, options \\ []) do - options = prepare_options(options) - apply(select_backend(options), :items, [entities, options]) + options = FrontendUtils.prepare_options(options) + apply(FrontendUtils.select_backend(options), :items, [entities, options]) end @doc """ @@ -175,8 +164,8 @@ defmodule Smee.Publish do """ @spec items_stream(entities :: Enumerable.t(), options :: keyword()) :: Enumerable.t(tuple()) def items_stream(entities, options \\ []) do - options = prepare_options(options) - apply(select_backend(options), :items_stream, [entities, options]) + options = FrontendUtils.prepare_options(options) + apply(FrontendUtils.select_backend(options), :items_stream, [entities, options]) end @doc """ @@ -190,8 +179,8 @@ defmodule Smee.Publish do """ @spec raw_stream(entities :: Enumerable.t(), options :: keyword()) :: Enumerable.t(tuple()) def raw_stream(entities, options \\ []) do - options = prepare_options(options) - apply(select_backend(options), :raw_stream, [entities, options]) + options = FrontendUtils.prepare_options(options) + apply(FrontendUtils.select_backend(options), :raw_stream, [entities, options]) end @doc """ @@ -203,8 +192,8 @@ defmodule Smee.Publish do """ @spec write_aggregate(entities :: Enumerable.t(), options :: keyword()) :: binary() def write_aggregate(entities, options \\ []) do - options = prepare_options(options) - apply(select_backend(options), :write_aggregate, [entities, options]) + options = FrontendUtils.prepare_options(options) + apply(FrontendUtils.select_backend(options), :write_aggregate, [entities, options]) end @doc """ @@ -221,31 +210,10 @@ defmodule Smee.Publish do """ @spec write_items(entities :: Enumerable.t(), options :: keyword()) :: list() def write_items(entities, options \\ []) do - options = prepare_options(options) - apply(select_backend(options), :write_items, [entities, options]) + options = FrontendUtils.prepare_options(options) + apply(FrontendUtils.select_backend(options), :write_items, [entities, options]) end ################################################################################ - defp prepare_options(options) do - Keyword.merge(@default_options, options) - |> Keyword.take(@allowed_options) - end - - defp select_backend(options) do - case options[:format] do - :csv -> Csv - :disco -> Disco - :index -> Index - :markdown -> Markdown - :metadata -> SamlXml - :saml -> SamlXml - :thiss -> Thiss - :udest -> Udest - :udisco -> Udisco - nil -> SamlXml - _ -> raise "Unknown publishing format '#{options[:format]} - known formats include #{formats()}'" - end - end - end diff --git a/lib/smee/publish/frontend_utils.ex b/lib/smee/publish/frontend_utils.ex new file mode 100644 index 0000000..7b8d2a6 --- /dev/null +++ b/lib/smee/publish/frontend_utils.ex @@ -0,0 +1,42 @@ +defmodule Smee.Publish.FrontendUtils do + + @default_options [format: :saml, lang: "en", id_type: :hash, to: "published", labels: false] + @allowed_options Keyword.keys(@default_options) ++ [:valid_until, :filename] + + @spec formats() :: list(atom()) + def formats() do + [ + :csv, + :disco, + :index, + :markdown, + :saml, + :thiss, + :udest, + :udisco + ] + end + + def prepare_options(options) do + Keyword.merge(@default_options, options) + |> Keyword.take(@allowed_options) + end + + def select_backend(options) do + case options[:format] do + :csv -> Smee.Publish.Csv + :disco -> Smee.Publish.Disco + :index -> Smee.Publish.Index + :markdown -> Smee.Publish.Markdown + :metadata -> Smee.Publish.SamlXml + :saml -> Smee.Publish.SamlXml + :thiss -> Smee.Publish.Thiss + :udest -> Smee.Publish.Udest + :udisco -> Smee.Publish.Udisco + :default -> Smee.Publish.SamlXml + nil -> Smee.Publish.SamlXml + _ -> raise "Unknown publishing format ':#{options[:format]}' - known formats include #{Enum.join(formats(), ", :")}" + end + end + +end diff --git a/test/smee/smee_publish_frontend_utils_test.exs b/test/smee/smee_publish_frontend_utils_test.exs new file mode 100644 index 0000000..0b0529d --- /dev/null +++ b/test/smee/smee_publish_frontend_utils_test.exs @@ -0,0 +1,81 @@ +defmodule SmeePublishFrontendUtilsTest do + use ExUnit.Case + + alias Smee.Publish.FrontendUtils + alias Smee.Source + #alias Smee.Metadata + #alias Smee.Lint + #alias Smee.XmlMunger + + @valid_metadata Source.new("test/support/static/aggregate.xml") + |> Smee.fetch!() + + describe "formats/0" do + + test "returns a list of supported publishing formats" do + assert [:csv, :disco, :index, :markdown, :saml, :thiss, :udest, :udisco] = FrontendUtils.formats() + end + + test "does not return a list containing private formats" do + refute Enum.member?(FrontendUtils.formats(), [:progress]) + refute Enum.member?(FrontendUtils.formats(), [:string]) + refute Enum.member?(FrontendUtils.formats(), [:null]) + end + + end + + describe "prepare_options/1" do + + test "user options have a format of :saml by default" do + assert :saml = Keyword.get(FrontendUtils.prepare_options([]), :format) + end + + test "user options have a lang of 'en' by default" do + assert "en" = Keyword.get(FrontendUtils.prepare_options([]), :lang) + end + + test "user options have an id_type of :hash by default" do + assert :hash = Keyword.get(FrontendUtils.prepare_options([]), :id_type) + end + + test "user options have a default output path of './published' by default" do + assert "published" = Keyword.get(FrontendUtils.prepare_options([]), :to) + end + + test "user options have index labels turned off by default" do + refute Keyword.get(FrontendUtils.prepare_options([]), :labels) + end + + test "unknown option keys do not pass through" do + refute Keyword.get(FrontendUtils.prepare_options([banana: "icecream"]), :banana) + end + + end + + describe "select_backend/1" do + + test "known, supported format types return a Publish module name" do + assert Smee.Publish.Csv = FrontendUtils.select_backend([format: :csv]) + assert Smee.Publish.Disco = FrontendUtils.select_backend([format: :disco]) + assert Smee.Publish.Index = FrontendUtils.select_backend([format: :index]) + assert Smee.Publish.Markdown = FrontendUtils.select_backend([format: :markdown]) + assert Smee.Publish.SamlXml = FrontendUtils.select_backend([format: :saml]) + assert Smee.Publish.Thiss = FrontendUtils.select_backend([format: :thiss]) + assert Smee.Publish.Udest = FrontendUtils.select_backend([format: :udest]) + assert Smee.Publish.Udisco = FrontendUtils.select_backend([format: :udisco]) + end + + test "there are various legacy aliases for the default format, SAML" do + assert Smee.Publish.SamlXml = FrontendUtils.select_backend([format: :metadata]) + assert Smee.Publish.SamlXml = FrontendUtils.select_backend([format: :nil]) + assert Smee.Publish.SamlXml = FrontendUtils.select_backend([format: :default]) + + end + + test "Unknown format types cause an exception" do + assert_raise RuntimeError, fn -> FrontendUtils.select_backend([format: :msword]) end + end + + end + +end diff --git a/test/smee/smee_publish_test.exs b/test/smee/smee_publish_test.exs index cda03f0..ffcad89 100644 --- a/test/smee/smee_publish_test.exs +++ b/test/smee/smee_publish_test.exs @@ -3,34 +3,183 @@ defmodule SmeePublishTest do alias Smee.Publish alias Smee.Source - #alias Smee.Metadata - #alias Smee.Lint - #alias Smee.XmlMunger + alias Smee.Metadata @valid_metadata Source.new("test/support/static/aggregate.xml") |> Smee.fetch!() + ## NOTE: These are mostly just smoke-tests to check that the functions here + ## are wired up to the right functions in the backend - the detailed tests + ## are elsewhere, for the backend functions + describe "formats/0" do test "returns a list of supported publishing formats" do assert [:csv, :disco, :index, :markdown, :saml, :thiss, :udest, :udisco] = Publish.formats() end + test "does not return a list containing private formats" do + refute Enum.member?(Publish.formats(), [:progress]) + refute Enum.member?(Publish.formats(), [:string]) + refute Enum.member?(Publish.formats(), [:null]) + end + + end + + describe "eslength/2" do + + test "returns the size of content in the stream for every format" do + + md_stream = Metadata.stream_entities(@valid_metadata) + + assert 41311 = Publish.eslength(md_stream) + assert 330 = Publish.eslength(md_stream, format: :csv) + assert 310 = Publish.eslength(md_stream, format: :disco) + assert 73 = Publish.eslength(md_stream, format: :index) + assert 464 = Publish.eslength(md_stream, format: :markdown) + assert 41311 = Publish.eslength(md_stream, format: :saml) + assert 434 = Publish.eslength(md_stream, format: :thiss) + assert 1746 = Publish.eslength(md_stream, format: :udest) + assert 147 = Publish.eslength(md_stream, format: :udisco) + end + + end + + describe "aggregate/2" do + + test "returns full text of aggregated items for every format" do + + md_stream = Metadata.stream_entities(@valid_metadata) + + assert " _ = Publish.aggregate(md_stream) + assert "http" <> _ = Publish.aggregate(md_stream, format: :csv) + assert "[" <> _ = Publish.aggregate(md_stream, format: :disco) + assert "http" <> _ = Publish.aggregate(md_stream, format: :index) + assert "|" <> _ = Publish.aggregate(md_stream, format: :markdown) + assert " _ = Publish.aggregate(md_stream, format: :saml) + assert "[" <> _ = Publish.aggregate(md_stream, format: :thiss) + assert "[" <> _ = Publish.aggregate(md_stream, format: :udest) + assert "[" <> _ = Publish.aggregate(md_stream, format: :udisco) + end + + end + + describe "items/2" do + + test "returns a map of keys and textually serialised items for every format" do + + md_stream = Metadata.stream_entities(@valid_metadata) + + assert is_map(Publish.items(md_stream)) + assert is_map(Publish.items(md_stream, format: :csv)) + assert is_map(Publish.items(md_stream, format: :disco)) + assert is_map(Publish.items(md_stream, format: :index)) + assert is_map(Publish.items(md_stream, format: :markdown)) + assert is_map(Publish.items(md_stream, format: :saml)) + assert is_map(Publish.items(md_stream, format: :thiss)) + assert is_map(Publish.items(md_stream, format: :udest)) + assert is_map(Publish.items(md_stream, format: :udisco)) + end + + end + + describe "aggregate_stream/2" do + + test "returns full text of aggregated items for every format, in a stream" do + + md_stream = Metadata.stream_entities(@valid_metadata) + + assert is_function(Publish.aggregate_stream(md_stream)) + assert is_function(Publish.aggregate_stream(md_stream, format: :csv)) + assert is_function(Publish.aggregate_stream(md_stream, format: :disco)) + assert is_function(Publish.aggregate_stream(md_stream, format: :index)) + assert is_function(Publish.aggregate_stream(md_stream, format: :markdown)) + assert is_function(Publish.aggregate_stream(md_stream, format: :saml)) + assert is_function(Publish.aggregate_stream(md_stream, format: :thiss)) + assert is_function(Publish.aggregate_stream(md_stream, format: :udest)) + assert is_function(Publish.aggregate_stream(md_stream, format: :udisco)) + end + + end + + describe "items_stream/2" do + + test "returns a map of keys and textually serialised items for every format" do + + md_stream = Metadata.stream_entities(@valid_metadata) + + assert 2 = Publish.items_stream(md_stream) |> Enum.count() + assert 2 = Publish.items_stream(md_stream, format: :csv) |> Enum.count() + assert 1 = Publish.items_stream(md_stream, format: :disco) |> Enum.count() + assert 2 = Publish.items_stream(md_stream, format: :index) |> Enum.count() + assert 2 = Publish.items_stream(md_stream, format: :markdown) |> Enum.count() + assert 2 = Publish.items_stream(md_stream, format: :saml) |> Enum.count() + assert 1 = Publish.items_stream(md_stream, format: :thiss) |> Enum.count() + assert 1 = Publish.items_stream(md_stream, format: :udest) |> Enum.count() + assert 1 = Publish.items_stream(md_stream, format: :udisco) |> Enum.count() + end + + end + + describe "raw_stream/2" do + + test "returns a stream of tuples of keys and JSON items for every format" do + + md_stream = Metadata.stream_entities(@valid_metadata) + + assert %Stream{} = Publish.raw_stream(md_stream) + assert %Stream{} = Publish.raw_stream(md_stream, format: :csv) + assert %Stream{} = Publish.raw_stream(md_stream, format: :disco) + assert %Stream{} = Publish.raw_stream(md_stream, format: :index) + assert %Stream{} = Publish.raw_stream(md_stream, format: :markdown) + assert %Stream{} = Publish.raw_stream(md_stream, format: :saml) + assert %Stream{} = Publish.raw_stream(md_stream, format: :thiss) + assert %Stream{} = Publish.raw_stream(md_stream, format: :udest) + assert %Stream{} = Publish.raw_stream(md_stream, format: :udisco) + end + end -# describe "eslength/2" do -# -# test "returns the size of content in the stream" do -# assert 330 = ThisModule.eslength(Metadata.stream_entities(@valid_metadata)) -# end -# -# test "should be about the same size as a compiled binary output" do -# actual_size = byte_size(ThisModule.aggregate(Metadata.stream_entities(@valid_metadata))) -# estimated_size = ThisModule.eslength(Metadata.stream_entities(@valid_metadata)) -# assert (actual_size - estimated_size) in -3..3 -# -# end -# -# end + + describe "write_aggregate/2" do + + @tag :tmp_dir + test "returns a filename for the aggregate written to disk", %{tmp_dir: tmp_dir} do + + md_stream = Metadata.stream_entities(@valid_metadata) + + assert File.exists?(Publish.write_aggregate(md_stream)) + assert File.exists?(Publish.write_aggregate(md_stream, format: :csv, to: tmp_dir)) + assert File.exists?(Publish.write_aggregate(md_stream, format: :disco, to: tmp_dir)) + assert File.exists?(Publish.write_aggregate(md_stream, format: :index, to: tmp_dir)) + assert File.exists?(Publish.write_aggregate(md_stream, format: :markdown, to: tmp_dir)) + assert File.exists?(Publish.write_aggregate(md_stream, format: :saml, to: tmp_dir)) + assert File.exists?(Publish.write_aggregate(md_stream, format: :thiss, to: tmp_dir)) + assert File.exists?(Publish.write_aggregate(md_stream, format: :udest, to: tmp_dir)) + assert File.exists?(Publish.write_aggregate(md_stream, format: :udisco, to: tmp_dir)) + end + + end + + + describe "write_items/2" do + + @tag :tmp_dir + test "returns a list of filenames after writing all items to disk", %{tmp_dir: tmp_dir} do + + md_stream = Metadata.stream_entities(@valid_metadata) + + assert Enum.all?(Publish.write_items(md_stream), &File.exists?/1) + assert Enum.all?(Publish.write_items(md_stream, format: :csv, to: tmp_dir), &File.exists?/1) + assert Enum.all?(Publish.write_items(md_stream, format: :disco, to: tmp_dir), &File.exists?/1) + assert Enum.all?(Publish.write_items(md_stream, format: :index, to: tmp_dir), &File.exists?/1) + assert Enum.all?(Publish.write_items(md_stream, format: :markdown, to: tmp_dir), &File.exists?/1) + assert Enum.all?(Publish.write_items(md_stream, format: :saml, to: tmp_dir), &File.exists?/1) + assert Enum.all?(Publish.write_items(md_stream, format: :thiss, to: tmp_dir), &File.exists?/1) + assert Enum.all?(Publish.write_items(md_stream, format: :udest, to: tmp_dir), &File.exists?/1) + assert Enum.all?(Publish.write_items(md_stream, format: :udisco, to: tmp_dir), &File.exists?/1) + end + + end end