From 0ba0ce7f9d9152ca62b89d6e15a3099c2215fbad Mon Sep 17 00:00:00 2001 From: Benjamin Lungwitz Date: Wed, 3 Feb 2021 19:35:02 +0100 Subject: [PATCH 01/11] added helper functions for cell dependencies --- src/analysis/CellDependencyVisualization.jl | 28 +++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 src/analysis/CellDependencyVisualization.jl diff --git a/src/analysis/CellDependencyVisualization.jl b/src/analysis/CellDependencyVisualization.jl new file mode 100644 index 0000000000..98f4465c2e --- /dev/null +++ b/src/analysis/CellDependencyVisualization.jl @@ -0,0 +1,28 @@ +""" +Gets the cell number in execution order (as saved in the notebook.jl file) +""" +get_cell_number(uuid:: UUID, notebook:: Notebook):: Int = findfirst(==(uuid),notebook.cell_order) +get_cell_number(cell:: Cell, notebook:: Notebook):: Int = get_cell_number(cell.cell_id, notebook) + +""" +Gets a list of all cells on which the current cell depends on. +Changes in these cells cause re-evaluation of the current cell. +""" +get_referenced_cells(cell:: Cell, notebook:: Notebook):: Vector{Cell} = Pluto.where_referenced(notebook, notebook.topology, cell) +get_referenced_cells(uuid:: UUID, notebook:: Notebook):: Vector{Cell} = get_referenced_cells(notebook.cells_dict[uuid], notebook) + +""" +Gets a list of all cells which are dependent on the current cell. +Changes in the current cell cause re-evaluation of these cells. +""" +function get_dependent_cells(cell:: Cell, notebook:: Notebook):: Vector{Cell} + node = notebook.topology.nodes[cell] + return Pluto.where_assigned(notebook, notebook.topology, node.references) +end +get_dependent_cells(uuid:: UUID, notebook:: Notebook):: Vector{Cell} = get_dependent_cells(notebook.cells_dict[uuid], notebook) + +"Converts a list of cells to a list of UUIDs." +get_cell_uuids(cells:: Vector{Cell}):: Vector{UUID} = getproperty.(cells, :cell_id) + +"Converts a list of cells to a list of execution order cell numbers." +get_cell_numbers(cells:: Vector{Cell}, notebook:: Notebook):: Vector{Int} = get_cell_number.(get_cell_uuids(cells), Ref(notebook)) From b1702f32fa0ae0fd81ba439066963301d0acd71f Mon Sep 17 00:00:00 2001 From: Benjamin Lungwitz Date: Mon, 31 May 2021 20:09:55 +0200 Subject: [PATCH 02/11] saving / loading disable status of cells --- src/notebook/Cell.jl | 3 +++ src/notebook/Notebook.jl | 33 ++++++++++++++++++++++++++++++++- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/src/notebook/Cell.jl b/src/notebook/Cell.jl index 1d981f3a68..97e3517cfe 100644 --- a/src/notebook/Cell.jl +++ b/src/notebook/Cell.jl @@ -41,6 +41,9 @@ Base.@kwdef mutable struct Cell running_disabled::Bool=false depends_on_disabled_cells::Bool=false + + running_disabled_on_startup::Bool=false + depends_on_disabled_cells_on_startup::Bool=false end Cell(cell_id, code) = Cell(cell_id=cell_id, code=code) diff --git a/src/notebook/Notebook.jl b/src/notebook/Notebook.jl index 8ea0363793..789bb59cac 100644 --- a/src/notebook/Notebook.jl +++ b/src/notebook/Notebook.jl @@ -78,6 +78,11 @@ const _order_delimiter = "# ā• ā•" const _order_delimiter_folded = "# ā•Ÿā”€" const _cell_suffix = "\n\n" +const _running_disabled_on_startup_prefix = "#= šŸ›‘ disabled at startup šŸ›‘" +const _running_disabled_on_startup_suffix = "\nšŸ›‘ disabled at startup šŸ›‘ =#" +const _depends_on_disabled_cells_on_startup_prefix = "#= šŸ›‘ depends on disabled cell(s) šŸ›‘" +const _depends_on_disabled_cells_on_startup_suffix = "\nšŸ›‘ depends on disabled cell(s) šŸ›‘ =#" + emptynotebook(args...) = Notebook([Cell()], args...) """ @@ -107,7 +112,21 @@ function save_notebook(io, notebook::Notebook) for c in cells_ordered println(io, _cell_id_delimiter, string(c.cell_id)) # write the cell code and prevent collisions with the cell delimiter - print(io, replace(c.code, _cell_id_delimiter => "# ")) + + if c.running_disabled_on_startup + println(io, _running_disabled_on_startup_prefix) + print(io, replace(c.code, _cell_id_delimiter => "# ")) + println(io, _running_disabled_on_startup_suffix) + elseif c.depends_on_disabled_cells_on_startup + # if a cell is both disabled directly and indirectly, the first has higher priority + println(io, _depends_on_disabled_cells_on_startup_prefix) + print(io, replace(c.code, _cell_id_delimiter => "# ")) + println(io, _depends_on_disabled_cells_on_startup_suffix) + else + # cell is not disabled on startup + print(io, replace(c.code, _cell_id_delimiter => "# ")) + end + print(io, _cell_suffix) end @@ -162,10 +181,22 @@ function load_notebook_nobackup(io, path)::Notebook code_raw = String(readuntil(io, _cell_id_delimiter)) # change Windows line endings to Linux code_normalised = replace(code_raw, "\r\n" => "\n") + + # get the information if a cell is directly disabled at startup + # the information if a cell is indirectly disabled is not fetched from the notebook file, + # but inferred in Pluto again. + running_disabled_on_startup = startswith(code_normalised, _running_disabled_on_startup_prefix) + + # remove the disabled on startup comments for further processing in Julia + code_normalised = replace(replace(code_normalised, _running_disabled_on_startup_prefix => ""), _running_disabled_on_startup_suffix => "") + code_normalised = replace(replace(code_normalised, _depends_on_disabled_cells_on_startup_prefix => ""), _depends_on_disabled_cells_on_startup_suffix => "") + # remove the cell suffix code = code_normalised[1:prevind(code_normalised, end, length(_cell_suffix))] read_cell = Cell(cell_id, code) + read_cell.running_disabled_on_startup = running_disabled_on_startup + read_cell.running_disabled = running_disabled_on_startup collected_cells[cell_id] = read_cell end end From 8342db6eab9941195ddabaf27531400587191746 Mon Sep 17 00:00:00 2001 From: Benjamin Lungwitz Date: Tue, 1 Jun 2021 13:23:31 +0200 Subject: [PATCH 03/11] no separate fields for deativated interactively and on startup --- src/notebook/Cell.jl | 3 --- src/notebook/Notebook.jl | 31 +++++++++++++++---------------- 2 files changed, 15 insertions(+), 19 deletions(-) diff --git a/src/notebook/Cell.jl b/src/notebook/Cell.jl index 97e3517cfe..1d981f3a68 100644 --- a/src/notebook/Cell.jl +++ b/src/notebook/Cell.jl @@ -41,9 +41,6 @@ Base.@kwdef mutable struct Cell running_disabled::Bool=false depends_on_disabled_cells::Bool=false - - running_disabled_on_startup::Bool=false - depends_on_disabled_cells_on_startup::Bool=false end Cell(cell_id, code) = Cell(cell_id=cell_id, code=code) diff --git a/src/notebook/Notebook.jl b/src/notebook/Notebook.jl index 789bb59cac..f230373541 100644 --- a/src/notebook/Notebook.jl +++ b/src/notebook/Notebook.jl @@ -78,10 +78,10 @@ const _order_delimiter = "# ā• ā•" const _order_delimiter_folded = "# ā•Ÿā”€" const _cell_suffix = "\n\n" -const _running_disabled_on_startup_prefix = "#= šŸ›‘ disabled at startup šŸ›‘" -const _running_disabled_on_startup_suffix = "\nšŸ›‘ disabled at startup šŸ›‘ =#" -const _depends_on_disabled_cells_on_startup_prefix = "#= šŸ›‘ depends on disabled cell(s) šŸ›‘" -const _depends_on_disabled_cells_on_startup_suffix = "\nšŸ›‘ depends on disabled cell(s) šŸ›‘ =#" +const _running_disabled_prefix = "#= šŸ›‘ disabled šŸ›‘" +const _running_disabled_suffix = "\nšŸ›‘ disabled šŸ›‘ =#" +const _depends_on_disabled_cells_prefix = "#= šŸ›‘ depends on disabled cell(s) šŸ›‘" +const _depends_on_disabled_cells_suffix = "\nšŸ›‘ depends on disabled cell(s) šŸ›‘ =#" emptynotebook(args...) = Notebook([Cell()], args...) @@ -113,15 +113,15 @@ function save_notebook(io, notebook::Notebook) println(io, _cell_id_delimiter, string(c.cell_id)) # write the cell code and prevent collisions with the cell delimiter - if c.running_disabled_on_startup - println(io, _running_disabled_on_startup_prefix) + if c.running_disabled + println(io, _running_disabled_prefix) print(io, replace(c.code, _cell_id_delimiter => "# ")) - println(io, _running_disabled_on_startup_suffix) - elseif c.depends_on_disabled_cells_on_startup + println(io, _running_disabled_suffix) + elseif c.depends_on_disabled_cells # if a cell is both disabled directly and indirectly, the first has higher priority - println(io, _depends_on_disabled_cells_on_startup_prefix) + println(io, _depends_on_disabled_cells_prefix) print(io, replace(c.code, _cell_id_delimiter => "# ")) - println(io, _depends_on_disabled_cells_on_startup_suffix) + println(io, _depends_on_disabled_cells_suffix) else # cell is not disabled on startup print(io, replace(c.code, _cell_id_delimiter => "# ")) @@ -182,21 +182,20 @@ function load_notebook_nobackup(io, path)::Notebook # change Windows line endings to Linux code_normalised = replace(code_raw, "\r\n" => "\n") - # get the information if a cell is directly disabled at startup + # get the information if a cell is directly disabled # the information if a cell is indirectly disabled is not fetched from the notebook file, # but inferred in Pluto again. - running_disabled_on_startup = startswith(code_normalised, _running_disabled_on_startup_prefix) + running_disabled = startswith(code_normalised, _running_disabled_prefix) # remove the disabled on startup comments for further processing in Julia - code_normalised = replace(replace(code_normalised, _running_disabled_on_startup_prefix => ""), _running_disabled_on_startup_suffix => "") - code_normalised = replace(replace(code_normalised, _depends_on_disabled_cells_on_startup_prefix => ""), _depends_on_disabled_cells_on_startup_suffix => "") + code_normalised = replace(replace(code_normalised, _running_disabled_prefix => ""), _running_disabled_suffix => "") + code_normalised = replace(replace(code_normalised, _depends_on_disabled_cells_prefix => ""), _depends_on_disabled_cells_suffix => "") # remove the cell suffix code = code_normalised[1:prevind(code_normalised, end, length(_cell_suffix))] read_cell = Cell(cell_id, code) - read_cell.running_disabled_on_startup = running_disabled_on_startup - read_cell.running_disabled = running_disabled_on_startup + read_cell.running_disabled = running_disabled collected_cells[cell_id] = read_cell end end From 36007f806e15f44503ba4c3eebd88cf9efaf315b Mon Sep 17 00:00:00 2001 From: Benjamin Lungwitz Date: Tue, 1 Jun 2021 14:14:38 +0200 Subject: [PATCH 04/11] extract disabling of dependent cells to function --- src/evaluation/Run.jl | 23 +++++++++++++++-------- src/notebook/Notebook.jl | 6 +++--- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/src/evaluation/Run.jl b/src/evaluation/Run.jl index c310bea0ac..24cae089a6 100644 --- a/src/evaluation/Run.jl +++ b/src/evaluation/Run.jl @@ -35,14 +35,7 @@ function run_reactive!(session::ServerSession, notebook::Notebook, old_topology: new_order = topological_order(notebook, new_topology, union(roots, keys(old_order.errable))) to_run_raw = setdiff(union(new_order.runnable, old_order.runnable), keys(new_order.errable))::Vector{Cell} # TODO: think if old error cell order matters - # find (indirectly) deactivated cells and update their status - deactivated = filter(c -> c.running_disabled, notebook.cells) - indirectly_deactivated = collect(topological_order(notebook, new_topology, deactivated)) - for cell in indirectly_deactivated - cell.running = false - cell.queued = false - cell.depends_on_disabled_cells = true - end + indirectly_deactivated = disable_dependent_cells!(notebook, new_topology) to_run = setdiff(to_run_raw, indirectly_deactivated) @@ -108,6 +101,20 @@ function run_reactive!(session::ServerSession, notebook::Notebook, old_topology: return new_order end +""" +find (indirectly) deactivated cells and update their status +""" +function disable_dependent_cells!(notebook:: Notebook, topology:: NotebookTopology) + deactivated = filter(c -> c.running_disabled, notebook.cells) + indirectly_deactivated = collect(topological_order(notebook, topology, deactivated)) + for cell in indirectly_deactivated + cell.running = false + cell.queued = false + cell.depends_on_disabled_cells = true + end + return indirectly_deactivated +end + run_reactive_async!(session::ServerSession, notebook::Notebook, to_run::Vector{Cell}; kwargs...) = run_reactive_async!(session, notebook, notebook.topology, notebook.topology, to_run; kwargs...) function run_reactive_async!(session::ServerSession, notebook::Notebook, old::NotebookTopology, new::NotebookTopology, to_run::Vector{Cell}; run_async::Bool=true, kwargs...) diff --git a/src/notebook/Notebook.jl b/src/notebook/Notebook.jl index f230373541..c53afb7770 100644 --- a/src/notebook/Notebook.jl +++ b/src/notebook/Notebook.jl @@ -182,10 +182,9 @@ function load_notebook_nobackup(io, path)::Notebook # change Windows line endings to Linux code_normalised = replace(code_raw, "\r\n" => "\n") - # get the information if a cell is directly disabled - # the information if a cell is indirectly disabled is not fetched from the notebook file, - # but inferred in Pluto again. + # get the information if a cell is disabled running_disabled = startswith(code_normalised, _running_disabled_prefix) + depends_on_disabled_cells = startswith(code_normalised, _depends_on_disabled_cells_prefix) # remove the disabled on startup comments for further processing in Julia code_normalised = replace(replace(code_normalised, _running_disabled_prefix => ""), _running_disabled_suffix => "") @@ -196,6 +195,7 @@ function load_notebook_nobackup(io, path)::Notebook read_cell = Cell(cell_id, code) read_cell.running_disabled = running_disabled + read_cell.depends_on_disabled_cells = depends_on_disabled_cells || running_disabled collected_cells[cell_id] = read_cell end end From da2d9d4dbdb3a5c3aee7e51137e5627f9e571c24 Mon Sep 17 00:00:00 2001 From: Benjamin Lungwitz Date: Tue, 1 Jun 2021 14:27:39 +0200 Subject: [PATCH 05/11] indirectly disabled cells saved correctly --- src/evaluation/Run.jl | 5 +++++ src/notebook/Notebook.jl | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/evaluation/Run.jl b/src/evaluation/Run.jl index 24cae089a6..3d4c17c4fa 100644 --- a/src/evaluation/Run.jl +++ b/src/evaluation/Run.jl @@ -105,6 +105,9 @@ end find (indirectly) deactivated cells and update their status """ function disable_dependent_cells!(notebook:: Notebook, topology:: NotebookTopology) + for cell in notebook.cells + cell.depends_on_disabled_cells = false + end deactivated = filter(c -> c.running_disabled, notebook.cells) indirectly_deactivated = collect(topological_order(notebook, topology, deactivated)) for cell in indirectly_deactivated @@ -184,6 +187,8 @@ function update_save_run!(session::ServerSession, notebook::Notebook, cells::Arr update_dependency_cache!(notebook) + disable_dependent_cells!(notebook, new) + session.options.server.disable_writing_notebook_files || save_notebook(notebook) # _assume `prerender_text == false` if you want to skip some details_ diff --git a/src/notebook/Notebook.jl b/src/notebook/Notebook.jl index c53afb7770..69363aa52c 100644 --- a/src/notebook/Notebook.jl +++ b/src/notebook/Notebook.jl @@ -187,8 +187,8 @@ function load_notebook_nobackup(io, path)::Notebook depends_on_disabled_cells = startswith(code_normalised, _depends_on_disabled_cells_prefix) # remove the disabled on startup comments for further processing in Julia - code_normalised = replace(replace(code_normalised, _running_disabled_prefix => ""), _running_disabled_suffix => "") - code_normalised = replace(replace(code_normalised, _depends_on_disabled_cells_prefix => ""), _depends_on_disabled_cells_suffix => "") + code_normalised = replace(replace(code_normalised, _running_disabled_prefix * "\n" => ""), _running_disabled_suffix * "\n" => "") + code_normalised = replace(replace(code_normalised, _depends_on_disabled_cells_prefix * "\n" => ""), _depends_on_disabled_cells_suffix * "\n" => "") # remove the cell suffix code = code_normalised[1:prevind(code_normalised, end, length(_cell_suffix))] From c2e0435e37ee6cdbb8d46d899390a10c998fcbb1 Mon Sep 17 00:00:00 2001 From: Benjamin Lungwitz Date: Tue, 1 Jun 2021 14:36:19 +0200 Subject: [PATCH 06/11] do not determine indirectly disabled cells twice --- src/evaluation/Run.jl | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/evaluation/Run.jl b/src/evaluation/Run.jl index 3d4c17c4fa..8f3ba2f47f 100644 --- a/src/evaluation/Run.jl +++ b/src/evaluation/Run.jl @@ -5,7 +5,8 @@ import .ExpressionExplorer: FunctionNameSignaturePair, is_joined_funcname, Using Base.push!(x::Set{Cell}) = x "Run given cells and all the cells that depend on them, based on the topology information before and after the changes." -function run_reactive!(session::ServerSession, notebook::Notebook, old_topology::NotebookTopology, new_topology::NotebookTopology, roots::Vector{Cell}; deletion_hook::Function=WorkspaceManager.delete_vars, persist_js_state::Bool=false)::TopologicalOrder +function run_reactive!(session::ServerSession, notebook::Notebook, old_topology::NotebookTopology, new_topology::NotebookTopology, roots::Vector{Cell}; + deletion_hook::Function=WorkspaceManager.delete_vars, persist_js_state::Bool=false, indirectly_deactivated::Union{Nothing, Vector{Cell}}=nothing)::TopologicalOrder # make sure that we're the only `run_reactive!` being executed - like a semaphor take!(notebook.executetoken) @@ -35,7 +36,9 @@ function run_reactive!(session::ServerSession, notebook::Notebook, old_topology: new_order = topological_order(notebook, new_topology, union(roots, keys(old_order.errable))) to_run_raw = setdiff(union(new_order.runnable, old_order.runnable), keys(new_order.errable))::Vector{Cell} # TODO: think if old error cell order matters - indirectly_deactivated = disable_dependent_cells!(notebook, new_topology) + if indirectly_deactivated === nothing + indirectly_deactivated = disable_dependent_cells!(notebook, new_topology) + end to_run = setdiff(to_run_raw, indirectly_deactivated) @@ -104,7 +107,7 @@ end """ find (indirectly) deactivated cells and update their status """ -function disable_dependent_cells!(notebook:: Notebook, topology:: NotebookTopology) +function disable_dependent_cells!(notebook:: Notebook, topology:: NotebookTopology):: Vector{Cell} for cell in notebook.cells cell.depends_on_disabled_cells = false end @@ -187,7 +190,7 @@ function update_save_run!(session::ServerSession, notebook::Notebook, cells::Arr update_dependency_cache!(notebook) - disable_dependent_cells!(notebook, new) + indirectly_deactivated = disable_dependent_cells!(notebook, new) session.options.server.disable_writing_notebook_files || save_notebook(notebook) @@ -218,7 +221,7 @@ function update_save_run!(session::ServerSession, notebook::Notebook, cells::Arr end if !(isempty(to_run_online) && session.options.evaluation.lazy_workspace_creation) && will_run_code(notebook) - run_reactive_async!(session, notebook, old, new, to_run_online; run_async=run_async, kwargs...) + run_reactive_async!(session, notebook, old, new, to_run_online; run_async=run_async, indirectly_deactivated=indirectly_deactivated, kwargs...) end end From a94685f172fdb80e76e87b9c1cbe899f148ad39d Mon Sep 17 00:00:00 2001 From: Benjamin Lungwitz Date: Wed, 2 Jun 2021 18:56:18 +0200 Subject: [PATCH 07/11] changed symbol for cell deactivation --- src/notebook/Notebook.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/notebook/Notebook.jl b/src/notebook/Notebook.jl index 69363aa52c..668f129a6e 100644 --- a/src/notebook/Notebook.jl +++ b/src/notebook/Notebook.jl @@ -78,10 +78,10 @@ const _order_delimiter = "# ā• ā•" const _order_delimiter_folded = "# ā•Ÿā”€" const _cell_suffix = "\n\n" -const _running_disabled_prefix = "#= šŸ›‘ disabled šŸ›‘" -const _running_disabled_suffix = "\nšŸ›‘ disabled šŸ›‘ =#" -const _depends_on_disabled_cells_prefix = "#= šŸ›‘ depends on disabled cell(s) šŸ›‘" -const _depends_on_disabled_cells_suffix = "\nšŸ›‘ depends on disabled cell(s) šŸ›‘ =#" +const _running_disabled_prefix = "#= ā• ā• disabled ā•ā•£" +const _running_disabled_suffix = "\nā• ā• disabled ā•ā•£ =#" +const _depends_on_disabled_cells_prefix = "#= ā• ā• depends on disabled cell(s) ā•ā•£" +const _depends_on_disabled_cells_suffix = "\nā• ā• depends on disabled cell(s) ā•ā•£ =#" emptynotebook(args...) = Notebook([Cell()], args...) From 471c5eebf83e4ed552a23f7684cc581cbfda7c34 Mon Sep 17 00:00:00 2001 From: Benjamin Lungwitz Date: Sun, 6 Jun 2021 20:24:52 +0200 Subject: [PATCH 08/11] disabling delimiter aligned --- src/notebook/Notebook.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/notebook/Notebook.jl b/src/notebook/Notebook.jl index 668f129a6e..fc2e8be903 100644 --- a/src/notebook/Notebook.jl +++ b/src/notebook/Notebook.jl @@ -78,10 +78,10 @@ const _order_delimiter = "# ā• ā•" const _order_delimiter_folded = "# ā•Ÿā”€" const _cell_suffix = "\n\n" -const _running_disabled_prefix = "#= ā• ā• disabled ā•ā•£" -const _running_disabled_suffix = "\nā• ā• disabled ā•ā•£ =#" -const _depends_on_disabled_cells_prefix = "#= ā• ā• depends on disabled cell(s) ā•ā•£" -const _depends_on_disabled_cells_suffix = "\nā• ā• depends on disabled cell(s) ā•ā•£ =#" +const _running_disabled_prefix = "#=ā• ā•ā•” disabled" +const _running_disabled_suffix = "\n ā• ā•ā•” disabled =#" +const _depends_on_disabled_cells_prefix = "#=ā• ā•ā•” depends on disabled cell(s)" +const _depends_on_disabled_cells_suffix = "\n ā• ā•ā•” depends on disabled cell(s) =#" emptynotebook(args...) = Notebook([Cell()], args...) From 639457c897e320f4909f5456786cd9766d53c9aa Mon Sep 17 00:00:00 2001 From: Benjamin Lungwitz Date: Mon, 7 Jun 2021 11:44:30 +0200 Subject: [PATCH 09/11] consistent line breaks --- src/notebook/Notebook.jl | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/notebook/Notebook.jl b/src/notebook/Notebook.jl index 4efb02b31d..f7768e04cd 100644 --- a/src/notebook/Notebook.jl +++ b/src/notebook/Notebook.jl @@ -73,14 +73,14 @@ end const _notebook_header = "### A Pluto.jl notebook ###" # We use a creative delimiter to avoid accidental use in code # so don't get inspired to suddenly use these in your code! -const _cell_id_delimiter = "# ā•”ā•ā•” " -const _order_delimiter = "# ā• ā•" -const _order_delimiter_folded = "# ā•Ÿā”€" +const _cell_id_delimiter = "# ā•”ā•ā•” " +const _order_delimiter = "# ā• ā•" +const _order_delimiter_folded = "# ā•Ÿā”€" const _cell_suffix = "\n\n" -const _running_disabled_prefix = "#=ā• ā•ā•” disabled" +const _running_disabled_prefix = "#=ā• ā•ā•” disabled\n" const _running_disabled_suffix = "\n ā• ā•ā•” disabled =#" -const _depends_on_disabled_cells_prefix = "#=ā• ā•ā•” depends on disabled cell(s)" +const _depends_on_disabled_cells_prefix = "#=ā• ā•ā•” depends on disabled cell(s)\n" const _depends_on_disabled_cells_suffix = "\n ā• ā•ā•” depends on disabled cell(s) =#" emptynotebook(args...) = Notebook([Cell()], args...) @@ -114,14 +114,14 @@ function save_notebook(io, notebook::Notebook) # write the cell code and prevent collisions with the cell delimiter if c.running_disabled - println(io, _running_disabled_prefix) + print(io, _running_disabled_prefix) print(io, replace(c.code, _cell_id_delimiter => "# ")) - println(io, _running_disabled_suffix) + print(io, _running_disabled_suffix) elseif c.depends_on_disabled_cells # if a cell is both disabled directly and indirectly, the first has higher priority - println(io, _depends_on_disabled_cells_prefix) + print(io, _depends_on_disabled_cells_prefix) print(io, replace(c.code, _cell_id_delimiter => "# ")) - println(io, _depends_on_disabled_cells_suffix) + print(io, _depends_on_disabled_cells_suffix) else # cell is not disabled on startup print(io, replace(c.code, _cell_id_delimiter => "# ")) @@ -187,8 +187,8 @@ function load_notebook_nobackup(io, path)::Notebook depends_on_disabled_cells = startswith(code_normalised, _depends_on_disabled_cells_prefix) # remove the disabled on startup comments for further processing in Julia - code_normalised = replace(replace(code_normalised, _running_disabled_prefix * "\n" => ""), _running_disabled_suffix * "\n" => "") - code_normalised = replace(replace(code_normalised, _depends_on_disabled_cells_prefix * "\n" => ""), _depends_on_disabled_cells_suffix * "\n" => "") + code_normalised = replace(replace(code_normalised, _running_disabled_prefix => ""), _running_disabled_suffix => "") + code_normalised = replace(replace(code_normalised, _depends_on_disabled_cells_prefix => ""), _depends_on_disabled_cells_suffix => "") # remove the cell suffix code = code_normalised[1:prevind(code_normalised, end, length(_cell_suffix))] From b8240ab97e00a004874e3c9edd05fc1d5392977b Mon Sep 17 00:00:00 2001 From: Benjamin Lungwitz Date: Mon, 19 Jul 2021 20:16:33 +0200 Subject: [PATCH 10/11] refactor cell disabling --- src/analysis/DependencyCache.jl | 20 +++++++++++++++-- src/evaluation/Run.jl | 39 ++++++--------------------------- src/notebook/Cell.jl | 4 ++-- src/notebook/Notebook.jl | 6 ++--- src/webserver/Dynamic.jl | 2 +- test/cell_disabling.jl | 6 ++--- 6 files changed, 33 insertions(+), 44 deletions(-) diff --git a/src/analysis/DependencyCache.jl b/src/analysis/DependencyCache.jl index 4be9fa5a25..a4bfd0ef69 100644 --- a/src/analysis/DependencyCache.jl +++ b/src/analysis/DependencyCache.jl @@ -33,13 +33,29 @@ function update_dependency_cache!(cell::Cell, notebook::Notebook) cell.cell_dependencies = CellDependencies( downstream_cells_map(cell, notebook), upstream_cells_map(cell, notebook), - cell_precedence_heuristic(notebook.topology, cell)) + cell_precedence_heuristic(notebook.topology, cell), + Ref(cell.running_disabled)) end "Fills dependency information on notebook and cell level." -function update_dependency_cache!(notebook::Notebook) +function update_dependency_cache!(notebook::Notebook, topology:: NotebookTopology) notebook._cached_topological_order = topological_order(notebook) for cell in values(notebook.cells_dict) update_dependency_cache!(cell, notebook) end + disable_dependent_cells!(notebook, topology) +end + +""" +find (indirectly) deactivated cells and update their status +""" +function disable_dependent_cells!(notebook:: Notebook, topology:: NotebookTopology):: Vector{Cell} + deactivated = filter(c -> c.running_disabled, notebook.cells) + indirectly_deactivated = collect(topological_order(notebook, topology, deactivated)) + for cell in indirectly_deactivated + cell.running = false + cell.queued = false + cell.cell_dependencies.depends_on_disabled_cells[] = true + end + return indirectly_deactivated end diff --git a/src/evaluation/Run.jl b/src/evaluation/Run.jl index b2fb1c1cf3..bbc1c2baaf 100644 --- a/src/evaluation/Run.jl +++ b/src/evaluation/Run.jl @@ -6,7 +6,7 @@ Base.push!(x::Set{Cell}) = x "Run given cells and all the cells that depend on them, based on the topology information before and after the changes." function run_reactive!(session::ServerSession, notebook::Notebook, old_topology::NotebookTopology, new_topology::NotebookTopology, roots::Vector{Cell}; - deletion_hook::Function=WorkspaceManager.delete_vars, persist_js_state::Bool=false, indirectly_deactivated::Union{Nothing, Vector{Cell}}=nothing)::TopologicalOrder + deletion_hook::Function=WorkspaceManager.delete_vars, persist_js_state::Bool=false)::TopologicalOrder # make sure that we're the only `run_reactive!` being executed - like a semaphor take!(notebook.executetoken) @@ -34,18 +34,12 @@ function run_reactive!(session::ServerSession, notebook::Notebook, old_topology: # get the new topological order new_order = topological_order(notebook, new_topology, union(roots, keys(old_order.errable))) - to_run_raw = setdiff(union(new_order.runnable, old_order.runnable), keys(new_order.errable))::Vector{Cell} # TODO: think if old error cell order matters - - if indirectly_deactivated === nothing - indirectly_deactivated = disable_dependent_cells!(notebook, new_topology) - end - - to_run = setdiff(to_run_raw, indirectly_deactivated) + to_run_including_deactivated = setdiff(union(new_order.runnable, old_order.runnable), keys(new_order.errable))::Vector{Cell} # TODO: think if old error cell order matters + to_run = filter(c -> !c.cell_dependencies.depends_on_disabled_cells[], to_run_including_deactivated) # change the bar on the sides of cells to "queued" - for cell in to_run - cell.queued = true - cell.depends_on_disabled_cells = false + for cell in to_run_including_deactivated + cell.queued = cell.running = !cell.cell_dependencies.depends_on_disabled_cells[] end for (cell, error) in new_order.errable cell.running = false @@ -104,23 +98,6 @@ function run_reactive!(session::ServerSession, notebook::Notebook, old_topology: return new_order end -""" -find (indirectly) deactivated cells and update their status -""" -function disable_dependent_cells!(notebook:: Notebook, topology:: NotebookTopology):: Vector{Cell} - for cell in notebook.cells - cell.depends_on_disabled_cells = false - end - deactivated = filter(c -> c.running_disabled, notebook.cells) - indirectly_deactivated = collect(topological_order(notebook, topology, deactivated)) - for cell in indirectly_deactivated - cell.running = false - cell.queued = false - cell.depends_on_disabled_cells = true - end - return indirectly_deactivated -end - run_reactive_async!(session::ServerSession, notebook::Notebook, to_run::Vector{Cell}; kwargs...) = run_reactive_async!(session, notebook, notebook.topology, notebook.topology, to_run; kwargs...) function run_reactive_async!(session::ServerSession, notebook::Notebook, old::NotebookTopology, new::NotebookTopology, to_run::Vector{Cell}; run_async::Bool=true, kwargs...) @@ -193,9 +170,7 @@ function update_save_run!(session::ServerSession, notebook::Notebook, cells::Arr old = notebook.topology new = notebook.topology = updated_topology(old, notebook, cells) - update_dependency_cache!(notebook) - - indirectly_deactivated = disable_dependent_cells!(notebook, new) + update_dependency_cache!(notebook, new) session.options.server.disable_writing_notebook_files || save_notebook(notebook) @@ -229,7 +204,7 @@ function update_save_run!(session::ServerSession, notebook::Notebook, cells::Arr sync_nbpkg(session, notebook; save=save) if !(isempty(to_run_online) && session.options.evaluation.lazy_workspace_creation) && will_run_code(notebook) # not async because that would be double async - run_reactive_async!(session, notebook, old, new, to_run_online; run_async=false, indirectly_deactivated=indirectly_deactivated, kwargs...) + run_reactive_async!(session, notebook, old, new, to_run_online; run_async=false, kwargs...) end end end diff --git a/src/notebook/Cell.jl b/src/notebook/Cell.jl index 01b9e82b3a..82b96907ca 100644 --- a/src/notebook/Cell.jl +++ b/src/notebook/Cell.jl @@ -17,6 +17,7 @@ struct CellDependencies{T} # T == Cell, but this has to be parametric to avoid a downstream_cells_map::Dict{Symbol,Vector{T}} upstream_cells_map::Dict{Symbol,Vector{T}} precedence_heuristic::Int + depends_on_disabled_cells:: Base.RefValue{Bool} # mutable value in immutable struct end "The building block of a `Notebook`. Contains code, output, reactivity data, mitochondria and ribosomes." @@ -37,10 +38,9 @@ Base.@kwdef mutable struct Cell runtime::Union{Nothing,UInt64}=nothing # note that this field might be moved somewhere else later. If you are interested in visualizing the cell dependencies, take a look at the cell_dependencies field in the frontend instead. - cell_dependencies::CellDependencies{Cell}=CellDependencies{Cell}(Dict{Symbol,Vector{Cell}}(), Dict{Symbol,Vector{Cell}}(), 99) + cell_dependencies::CellDependencies{Cell}=CellDependencies{Cell}(Dict{Symbol,Vector{Cell}}(), Dict{Symbol,Vector{Cell}}(), 99, Ref(false)) running_disabled::Bool=false - depends_on_disabled_cells::Bool=false end Cell(cell_id, code) = Cell(cell_id=cell_id, code=code) diff --git a/src/notebook/Notebook.jl b/src/notebook/Notebook.jl index 9bcde7b3cd..e55d9deffa 100644 --- a/src/notebook/Notebook.jl +++ b/src/notebook/Notebook.jl @@ -130,7 +130,7 @@ function save_notebook(io, notebook::Notebook) print(io, _running_disabled_prefix) print(io, replace(c.code, _cell_id_delimiter => "# ")) print(io, _running_disabled_suffix) - elseif c.depends_on_disabled_cells + elseif c.cell_dependencies.depends_on_disabled_cells[] # if a cell is both disabled directly and indirectly, the first has higher priority print(io, _depends_on_disabled_cells_prefix) print(io, replace(c.code, _cell_id_delimiter => "# ")) @@ -232,7 +232,6 @@ function load_notebook_nobackup(io, path)::Notebook # get the information if a cell is disabled running_disabled = startswith(code_normalised, _running_disabled_prefix) - depends_on_disabled_cells = startswith(code_normalised, _depends_on_disabled_cells_prefix) # remove the disabled on startup comments for further processing in Julia code_normalised = replace(replace(code_normalised, _running_disabled_prefix => ""), _running_disabled_suffix => "") @@ -243,7 +242,6 @@ function load_notebook_nobackup(io, path)::Notebook read_cell = Cell(cell_id, code) read_cell.running_disabled = running_disabled - read_cell.depends_on_disabled_cells = depends_on_disabled_cells || running_disabled collected_cells[cell_id] = read_cell end end @@ -328,7 +326,7 @@ function load_notebook(path::String; disable_writing_notebook_files::Bool=false) loaded = load_notebook_nobackup(path) # Analyze cells so that the initial save is in topological order loaded.topology = updated_topology(loaded.topology, loaded, loaded.cells) - update_dependency_cache!(loaded) + update_dependency_cache!(loaded, loaded.topology) disable_writing_notebook_files || save_notebook(loaded) loaded.topology = NotebookTopology() diff --git a/src/webserver/Dynamic.jl b/src/webserver/Dynamic.jl index 7868a47d0c..dced2c2471 100644 --- a/src/webserver/Dynamic.jl +++ b/src/webserver/Dynamic.jl @@ -124,7 +124,7 @@ function notebook_to_js(notebook::Notebook) "cell_results" => Dict{UUID,Dict{String,Any}}( id => Dict{String,Any}( "cell_id" => cell.cell_id, - "depends_on_disabled_cells" => cell.depends_on_disabled_cells, + "depends_on_disabled_cells" => cell.cell_dependencies.depends_on_disabled_cells[], "output" => Dict( "body" => cell.output.body, "mime" => cell.output.mime, diff --git a/test/cell_disabling.jl b/test/cell_disabling.jl index 7e1eb83a72..f8de95cb02 100644 --- a/test/cell_disabling.jl +++ b/test/cell_disabling.jl @@ -24,10 +24,10 @@ using Pluto: update_run!, ServerSession, ClientSession, Cell, Notebook # helper functions id(i) = notebook.cells[i].cell_id - get_disabled_cells(notebook) = [i for (i, c) in pairs(notebook.cells) if c.depends_on_disabled_cells] + get_disabled_cells(notebook) = [i for (i, c) in pairs(notebook.cells) if c.cell_dependencies.depends_on_disabled_cells[]] @test !any(c.running_disabled for c in notebook.cells) - @test !any(c.depends_on_disabled_cells for c in notebook.cells) + @test !any(c.cell_dependencies.depends_on_disabled_cells[] for c in notebook.cells) # disable first cell notebook.cells[1].running_disabled = true @@ -63,7 +63,7 @@ using Pluto: update_run!, ServerSession, ClientSession, Cell, Notebook original_6_output = notebook.cells[6].output.body setcode(notebook.cells[6], "x + 6") update_run!(šŸ­, notebook, notebook.cells[6]) - @test notebook.cells[6].depends_on_disabled_cells + @test notebook.cells[6].cell_dependencies.depends_on_disabled_cells[] @test notebook.cells[6].errored === false @test notebook.cells[6].output.body == original_6_output From 891b110e17bd83c275e4454b0a5f4247f81e197b Mon Sep 17 00:00:00 2001 From: Benjamin Lungwitz Date: Mon, 19 Jul 2021 20:30:21 +0200 Subject: [PATCH 11/11] add tests that notebook is executable --- test/cell_disabling.jl | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/test/cell_disabling.jl b/test/cell_disabling.jl index f8de95cb02..676922d679 100644 --- a/test/cell_disabling.jl +++ b/test/cell_disabling.jl @@ -1,6 +1,6 @@ using Test using Pluto -using Pluto: update_run!, ServerSession, ClientSession, Cell, Notebook +using Pluto: update_run!, ServerSession, ClientSession, Cell, Notebook, save_notebook @testset "Cell Disabling" begin šŸ­ = ServerSession() @@ -22,6 +22,8 @@ using Pluto: update_run!, ServerSession, ClientSession, Cell, Notebook fakeclient.connected_notebook = notebook update_run!(šŸ­, notebook, notebook.cells) + notebook_file = tempname() + # helper functions id(i) = notebook.cells[i].cell_id get_disabled_cells(notebook) = [i for (i, c) in pairs(notebook.cells) if c.cell_dependencies.depends_on_disabled_cells[]] @@ -29,11 +31,16 @@ using Pluto: update_run!, ServerSession, ClientSession, Cell, Notebook @test !any(c.running_disabled for c in notebook.cells) @test !any(c.cell_dependencies.depends_on_disabled_cells[] for c in notebook.cells) + save_notebook(notebook, notebook_file) + @test jl_is_runnable(notebook_file) + # disable first cell notebook.cells[1].running_disabled = true update_run!(šŸ­, notebook, notebook.cells) should_be_disabled = [1, 3, 5] @test get_disabled_cells(notebook) == should_be_disabled + save_notebook(notebook, notebook_file) + @test jl_is_runnable(notebook_file) # change x, this change should not propagate through y original_y_output = notebook.cells[1].output.body @@ -46,6 +53,8 @@ using Pluto: update_run!, ServerSession, ClientSession, Cell, Notebook @test notebook.cells[3].output.body == original_z_output @test notebook.cells[4].output.body != original_a_output @test notebook.cells[5].output.body == original_w_output + save_notebook(notebook, notebook_file) + @test jl_is_runnable(notebook_file) setcode(notebook.cells[2], "x = 2") update_run!(šŸ­, notebook, notebook.cells[2]) @@ -53,12 +62,15 @@ using Pluto: update_run!, ServerSession, ClientSession, Cell, Notebook @test notebook.cells[3].output.body == original_z_output @test notebook.cells[4].output.body == original_a_output @test notebook.cells[5].output.body == original_w_output + save_notebook(notebook, notebook_file) + @test jl_is_runnable(notebook_file) # disable root cell notebook.cells[2].running_disabled = true update_run!(šŸ­, notebook, notebook.cells) @test get_disabled_cells(notebook) == collect(1:5) - + save_notebook(notebook, notebook_file) + @test jl_is_runnable(notebook_file) original_6_output = notebook.cells[6].output.body setcode(notebook.cells[6], "x + 6") @@ -66,11 +78,15 @@ using Pluto: update_run!, ServerSession, ClientSession, Cell, Notebook @test notebook.cells[6].cell_dependencies.depends_on_disabled_cells[] @test notebook.cells[6].errored === false @test notebook.cells[6].output.body == original_6_output + save_notebook(notebook, notebook_file) + @test jl_is_runnable(notebook_file) # reactivate first cell - still all cells should be running_disabled notebook.cells[1].running_disabled = false update_run!(šŸ­, notebook, notebook.cells) @test get_disabled_cells(notebook) == collect(1:6) + save_notebook(notebook, notebook_file) + @test jl_is_runnable(notebook_file) # the x cell is disabled, so changing it should have no effect setcode(notebook.cells[2], "x = 123123") @@ -79,27 +95,34 @@ using Pluto: update_run!, ServerSession, ClientSession, Cell, Notebook @test notebook.cells[3].output.body == original_z_output @test notebook.cells[4].output.body == original_a_output @test notebook.cells[5].output.body == original_w_output + save_notebook(notebook, notebook_file) + @test jl_is_runnable(notebook_file) # reactivate root cell notebook.cells[2].running_disabled = false update_run!(šŸ­, notebook, notebook.cells) @test get_disabled_cells(notebook) == [] - @test notebook.cells[1].output.body != original_y_output @test notebook.cells[3].output.body != original_z_output @test notebook.cells[4].output.body != original_a_output @test notebook.cells[5].output.body != original_w_output + save_notebook(notebook, notebook_file) + @test jl_is_runnable(notebook_file) # disable first cell again notebook.cells[1].running_disabled = true update_run!(šŸ­, notebook, notebook.cells) should_be_disabled = [1, 3, 5] @test get_disabled_cells(notebook) == should_be_disabled + save_notebook(notebook, notebook_file) + @test jl_is_runnable(notebook_file) # and reactivate it notebook.cells[1].running_disabled = false update_run!(šŸ­, notebook, notebook.cells) @test get_disabled_cells(notebook) == [] + save_notebook(notebook, notebook_file) + @test jl_is_runnable(notebook_file) end