Merge pull request #194 from JuliaAI/measure

Update code and tests to address migration of measures MLJBase -> StatisticalMeasures

Project.toml

@@ -1,7 +1,7 @@
 name = "MLJTuning"
 uuid = "03970b2e-30c4-11ea-3135-d1576263f10f"
 authors = ["Anthony D. Blaom <[email protected]>"]
-version = "0.7.4"
+version = "0.8.0"
 
 [deps]
 ComputationalResources = "ed09eef8-17a6-5b46-8889-db040fac31e3"
@@ -12,15 +12,16 @@ MLJBase = "a7f614a8-145f-11e9-1d2a-a57a1082229d"
 ProgressMeter = "92933f4c-e287-5a05-a399-4b506db050ca"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 RecipesBase = "3cdcf5f2-1ef4-517c-9805-6587b60abb01"
+StatisticalMeasuresBase = "c062fc1d-0d66-479b-b6ac-8b44719de4cc"
 
 [compat]
 ComputationalResources = "0.3"
 Distributions = "0.22,0.23,0.24, 0.25"
 LatinHypercubeSampling = "1.7.2"
-MLJBase = "0.20, 0.21"
+MLJBase = "1"
 ProgressMeter = "1.7.1"
 RecipesBase = "0.8,0.9,1"
-julia = "1.6"
+StatisticalMeasuresBase = "0.1.1"
 
 [extras]
 CategoricalArrays = "324d7699-5711-5eae-9e2f-1d82baa6b597"
@@ -35,11 +36,11 @@ NearestNeighbors = "b8a86587-4115-5ab1-83bc-aa920d37bbce"
 ScientificTypes = "321657f4-b219-11e9-178b-2701a2544e81"
 Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
 StableRNGs = "860ef19b-820b-49d6-a774-d7a799459cd3"
+StatisticalMeasures = "a19d573c-0a75-4610-95b3-7071388c7541"
 Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
 Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [targets]
-test = ["CategoricalArrays", "ComputationalResources", "DecisionTree", "Distances", "Distributed", "LinearAlgebra", "MLJModelInterface", "MultivariateStats",  "NearestNeighbors", "ScientificTypes",  "Serialization",  "StableRNGs", "Statistics", "StatsBase",  "Tables", "Test"]
-
+test = ["CategoricalArrays", "ComputationalResources", "DecisionTree", "Distances", "Distributed", "LinearAlgebra", "MLJModelInterface", "MultivariateStats", "NearestNeighbors", "ScientificTypes", "Serialization", "StableRNGs", "StatisticalMeasures", "Statistics", "StatsBase", "Tables", "Test"]

src/MLJTuning.jl

@@ -23,6 +23,7 @@ using MLJBase
 import MLJBase: Bounded, Unbounded, DoublyUnbounded,
     LeftUnbounded, RightUnbounded, _process_accel_settings, chunks,
     restore, save
+import StatisticalMeasuresBase as SMB
 using RecipesBase
 using Distributed
 import Distributions

src/tuned_models.jl

@@ -3,7 +3,7 @@
 const ERR_SPECIFY_MODEL = ArgumentError(
 "You need to specify `model=...`, unless `tuning=Explicit()`. ")
 const ERR_SPECIFY_RANGE = ArgumentError(
-    "You need to specify `range=...`, unless `tuning=Explicit` and "*
+    "You need to specify `range=...`, unless `tuning=Explicit()` and "*
     "and `models=...` is specified instead. ")
 const ERR_SPECIFY_RANGE_OR_MODELS = ArgumentError(
     "No `model` specified. Either specify an explicit iterator "*
@@ -12,7 +12,7 @@ const ERR_SPECIFY_RANGE_OR_MODELS = ArgumentError(
 const ERR_NEED_EXPLICIT = ArgumentError(
     "You have specified an explicit "*
     "iterator `models` of MLJModels and so cannot "*
-    "specify any `tuning` strategy except `Explicit`. Either omit the "*
+    "specify any `tuning` strategy except `Explicit()`. Either omit the "*
     "`tuning=...` specification, or specify a *single* model using "*
     "`model=...` instead. ")
 const ERR_BOTH_DISALLOWED = ArgumentError(

src/utilities.jl

@@ -30,10 +30,11 @@ function delete(nt::NamedTuple, target_keys...)
     return (; filtered...)
 end
 
+# `SMB` is alias for `StatisticalMeasuresBase`
 signature(measure) =
-    if orientation(measure) == :loss
+    if SMB.orientation(measure) == SMB.Loss()
         1
-    elseif orientation(measure) == :score
+    elseif SMB.orientation(measure) == SMB.Score()
         -1
     else
         0

test/runtests.jl

@@ -4,6 +4,7 @@ addprocs(2)
 using Test
 using MLJTuning
 using MLJBase
+using StatisticalMeasures
 using StableRNGs
 
 # Display Number of processes and if necessary number

test/strategies/grid.jl

@@ -2,6 +2,7 @@ module TestGrid
 
 using Test
 using MLJBase
+using StatisticalMeasures
 using MLJTuning
 # include("../test/models.jl")
 # using .Models

test/strategies/latin_hypercube.jl

@@ -3,6 +3,7 @@ module TestLatinHypercube
 using Test
 using MLJBase
 using MLJTuning
+using StatisticalMeasures
 using LatinHypercubeSampling
 import Distributions
 import Random

test/strategies/random_search.jl

@@ -2,6 +2,7 @@ module TestRandomSearch
 
 using Test
 using MLJBase
+using StatisticalMeasures
 using MLJTuning
 import Distributions
 import Random

test/tuned_models.jl

@@ -29,41 +29,41 @@ r = [m(K) for K in 13:-1:2]
 
 @testset "constructor" begin
     @test_throws(MLJTuning.ERR_SPECIFY_RANGE,
-                 TunedModel(model=first(r), tuning=Grid(), measure=rms))
+                 TunedModel(model=first(r), tuning=Grid(), measure=l2))
     @test_throws(MLJTuning.ERR_SPECIFY_RANGE,
-                 TunedModel(model=first(r), measure=rms))
+                 TunedModel(model=first(r), measure=l2))
     @test_throws(MLJTuning.ERR_BOTH_DISALLOWED,
                  TunedModel(model=first(r),
-                            models=r, tuning=Explicit(), measure=rms))
-    tm = @test_logs TunedModel(models=r, tuning=Explicit(), measure=rms)
+                            models=r, tuning=Explicit(), measure=l2))
+    tm = @test_logs TunedModel(models=r, tuning=Explicit(), measure=l2)
     @test tm.tuning isa Explicit && tm.range ==r && tm.model == first(r)
     @test input_scitype(tm) == Unknown
-    @test TunedModel(models=r, measure=rms) == tm
+    @test TunedModel(models=r, measure=l2) == tm
     @test_logs (:info, r"No measure") @test TunedModel(models=r) == tm
 
     @test_throws(MLJTuning.ERR_SPECIFY_MODEL,
-                 TunedModel(range=r, measure=rms))
+                 TunedModel(range=r, measure=l2))
     @test_throws(MLJTuning.ERR_MODEL_TYPE,
                  TunedModel(model=42, tuning=Grid(),
-                            range=r, measure=rms))
+                            range=r, measure=l2))
     @test_logs (:info, MLJTuning.INFO_MODEL_IGNORED) tm =
-        TunedModel(model=42, tuning=Explicit(), range=r, measure=rms)
+        TunedModel(model=42, tuning=Explicit(), range=r, measure=l2)
     @test_logs (:info, r"No measure") tm =
         TunedModel(model=first(r), range=r)
     @test_throws(MLJTuning.ERR_SPECIFY_RANGE_OR_MODELS,
-                TunedModel(tuning=Explicit(), measure=rms))
+                TunedModel(tuning=Explicit(), measure=l2))
     @test_throws(MLJTuning.ERR_NEED_EXPLICIT,
                  TunedModel(models=r, tuning=Grid()))
-    @test_logs TunedModel(first(r), range=r, measure=rms)
+    @test_logs TunedModel(first(r), range=r, measure=l2)
     @test_logs(
         (:warn, MLJTuning.warn_double_spec(first(r), last(r))),
-        TunedModel(first(r), model=last(r), range=r, measure=rms),
+        TunedModel(first(r), model=last(r), range=r, measure=l2),
     )
     @test_throws(
         MLJTuning.ERR_TOO_MANY_ARGUMENTS,
-        TunedModel(first(r), last(r), range=r, measure=rms),
+        TunedModel(first(r), last(r), range=r, measure=l2),
     )
-    tm = @test_logs TunedModel(model=first(r), range=r, measure=rms)
+    tm = @test_logs TunedModel(model=first(r), range=r, measure=l2)
     @test tm.tuning isa RandomSearch
     @test input_scitype(tm) == Table(Continuous)