Merge pull request #270 from JuliaCI/tb/julia_test_args

Forward Julia arguments to Pkg.test

bin/test_package.jl

@@ -5,7 +5,7 @@ function usage(error=nothing)
         println(stderr, "ERROR: $error")
     end
     println(stderr, """
-        Usage: julia test_package.jl [--julia=nightly] [--rr=false]
+        Usage: julia test_package.jl [--julia=nightly] [--julia_args=""] [--env=""] [--rr=false]
                                      [--name=...] [--version=...] [--rev=...] [--url=...] [--path=...]
 
         This script can be used to quickly test a package against a specific version of Julia.
@@ -14,7 +14,9 @@ function usage(error=nothing)
         To test a local development version, use the `--path` flag.
 
         The `--julia` flag can be used to specify the version of Julia to test with, and defaults to `nightly`.
-        With the `--rr` flag you can enable running under `rr`, as used by daily PkgEval runs.""")
+        To pass additional arguments to Julia, use one or more `--julia_args` flag.
+        Similarly, to set environment variables, use one or more `--env` flag.
+        With the `--rr` flag you can enable running under `rr`.""")
     exit(error === nothing ? 0 : 1)
 end
 
@@ -26,28 +28,48 @@ end
 
 args = Dict()
 for arg in ARGS
-    startswith(arg, "--") || usage("invalid argument: $arg")
-    contains(arg, "=")    || usage("invalid argument: $arg")
+    startswith(arg, "--") || usage("unknown argument: $arg")
+    contains(arg, "=")    || usage("argument missing value: $arg")
 
     option, value = split(arg, "="; limit=2)
-    args[Symbol(option[3:end])] = String(value)
+    flag = Symbol(option[3:end])
+    if haskey(args, flag)
+        push!(args[flag], String(value))
+    else
+        args[flag] = [String(value)]
+    end
 end
 
 # create the Configuration object
-config_flags = [(:julia => String), (:rr => Bool)]
+config_flags = [(:julia => String), (:julia_args => Vector{String}),
+                (:env => Vector{String}), (:rr => Bool)]
 config_args = Dict()
+function parse_value(typ, val)
+    if typ === String
+        val
+    else
+        parse(typ, val)
+    end
+end
 for (flag, typ) in config_flags
     if haskey(args, flag)
-        config_args[flag] = if typ === String
-            args[flag]
+        config_args[flag] = if typ <: Vector
+            parse_value.(Ref(eltype(typ)), args[flag])
         else
-            parse(typ, args[flag])
+            length(args[flag]) == 1 || usage("multiple values for --$flag")
+            parse_value(typ, only(args[flag]))
         end
         delete!(args, flag)
     end
 end
 config = Configuration(; config_args...)
 
+# remaining arguments should be singular
+for flag in keys(args)
+    length(args[flag]) == 1 || usage("multiple values for --$flag")
+end
+args = Dict(key => only(val) for (key, val) in args)
+
 result = if haskey(args, :path)
     path = expanduser(args[:path])
     delete!(args, :path)

scripts/compile.jl

@@ -12,7 +12,7 @@ versioninfo()
 
 
 print("\n\n", '#'^80, "\n# Installation\n#\n\n")
-t0 = time()
+t0 = cpu_time()
 
 is_stdlib = any(Pkg.Types.stdlibs()) do (uuid,stdlib)
     name = isa(stdlib, String) ? stdlib : first(stdlib)
@@ -35,7 +35,7 @@ println("\nCompleted after $(elapsed(t0))")
 
 
 print("\n\n", '#'^80, "\n# Compilation\n#\n\n")
-t1 = time()
+t1 = cpu_time()
 
 create_sysimage([pkg.name]; sysimage_path)
 println("\nCompleted after $(elapsed(t1))")

scripts/test.jl

@@ -43,27 +43,33 @@ end
 # generating package images is really expensive, without much benefit (for PkgEval)
 # so determine here if we need to disable them using additional CLI args
 # (we can't do this externally because of JuliaLang/Pkg.jl#3737)
-julia_args = if VERSION < v"1.9-beta1" || (v"1.10-" <= VERSION < v"1.10.0-DEV.204")
+julia_args = String[]
+if VERSION < v"1.9-beta1" || (v"1.10-" <= VERSION < v"1.10.0-DEV.204")
     # we don't support pkgimages yet
-    ``
-elseif any(startswith("--pkgimages"), config.julia_flags)
+elseif any(startswith("--pkgimages"), config.julia_args)
     # the user specifically requested pkgimages
-    ``
 else
     if VERSION >= v"1.11-DEV.1119"
         # we can selectively disable pkgimages while allowing reuse of existing ones
-        `--pkgimages=existing`
+        push!(julia_args, "--pkgimages=existing")
     elseif VERSION >= v"1.11-DEV.123"
         # we can only selectively disable all compilation caches. this isn't ideal,
         # but at this point in time (where many stdlibs have been moved out of the
         # system image) it's strictly better than using `--pkgimages=no`
-        `--compiled-modules=existing`
+        push!(julia_args, "--compiled-modules=existing")
     else
         # completely disable pkgimages
-        `--pkgimages=no`
+        push!(julia_args, "--pkgimages=no")
     end
 end
 
+# forward all user-provided flags to `Pkg.test` to ensure the test environment uses them.
+# we need to do so because Pkg inserts its own flags we want to be able to override (e.g.,
+# `--check-bounds`), and because its use of `Base.julia_cmd()` doesn't include all flags
+# passed to the current Julia process (e.g. `--inline`).
+append!(julia_args, config.julia_args)
+julia_args = Cmd(julia_args)
+
 io = IOBuffer()
 Pkg.DEFAULT_IO[] = io
 try

src/PkgEvalCore.jl

@@ -57,7 +57,7 @@ Base.@kwdef struct Configuration
     julia_install_dir::Setting{String} = Default("/opt/julia")
     julia_binary::Setting{String} = Default("julia")
     ## additional Julia arguments to pass to the process
-    julia_flags::Setting{Vector{String}} = Default(String[])
+    julia_args::Setting{Vector{String}} = Default(String[])
 
     # registry properties
     ## the repo spec of the registry to use
@@ -132,7 +132,7 @@ function Base.show(io::IO, ::MIME"text/plain", cfg::Configuration)
     println(io, "PkgEval configuration '$(cfg.name)' (")
 
     println(io, "  # Julia properties")
-    show_setting.(["julia", "buildflags", "buildcommands", "julia_install_dir", "julia_binary", "julia_flags"])
+    show_setting.(["julia", "buildflags", "buildcommands", "julia_install_dir", "julia_binary", "julia_args"])
     println(io)
 
     println(io, "  # Registry properties")

src/evaluate.jl

@@ -609,7 +609,7 @@ function evaluate_compiled_test(config::Configuration, pkg::Package;
     compile_log = log
     test_config = Configuration(config;
         compiled = false,
-        julia_flags = [config.julia_flags..., "--sysimage", sysimage_path],
+        julia_args = [config.julia_args..., "--sysimage", sysimage_path],
     )
     (; log, status, reason, version, duration, input_output) =
         evaluate_test(test_config, pkg; mounts, use_cache, kwargs...)

src/sandbox.jl

@@ -368,7 +368,7 @@ function setup_julia_sandbox(config::Configuration, args=``;
     # configure threads
     env["JULIA_NUM_THREADS"] = string(config.threads)
 
-    setup_generic_sandbox(config, `$cmd $(Cmd(config.julia_flags)) $args`;
+    setup_generic_sandbox(config, `$cmd $(Cmd(config.julia_args)) $args`;
                           env, mounts, kwargs...)
 end
 

test/runtests.jl

@@ -148,7 +148,7 @@ end
 
 if julia_version >= v"1.10.0-DEV.204" || v"1.9.0-alpha1.55" <= julia_version < v"1.10-"
 @testset "package precompilation" begin
-    let config = Configuration(config; julia_flags=["--pkgimages=yes"])
+    let config = Configuration(config; julia_args=["--pkgimages=yes"])
         # find out where Example.jl will be precompiled
         verstr = "v$(julia_version.major).$(julia_version.minor)"
         compilecache = joinpath(PkgEval.get_compilecache(config), verstr, "Example")