From 4ce8831bda3d528c4c30876780a2fdcc5c9ceb8c Mon Sep 17 00:00:00 2001 From: Derek Gerstmann Date: Thu, 28 Sep 2023 14:30:44 -0700 Subject: [PATCH] [serialization] Add support to serialize to memory, and a basic serialization tutorial (#7760) * Add in-memory buffer serialize/deserialize support. * Add basic serialization tutorial * Clang format pass * Update doc strings to use Doxygen formatted args * Clear out data buffer during serialization * Update serialization tutorial to use simple blur example with ImageParam * Make parameter map optional for serialize #7849 Add error messages to deserializer for missing params Update tutorial * Clang format pass --------- Co-authored-by: Derek Gerstmann Co-authored-by: Steven Johnson --- src/Deserialization.cpp | 36 +++++++- src/Deserialization.h | 20 +++-- src/Serialization.cpp | 49 +++++++++- src/Serialization.h | 21 +++++ tutorial/CMakeLists.txt | 3 +- tutorial/lesson_23_serialization.cpp | 130 +++++++++++++++++++++++++++ 6 files changed, 248 insertions(+), 11 deletions(-) create mode 100644 tutorial/lesson_23_serialization.cpp diff --git a/src/Deserialization.cpp b/src/Deserialization.cpp index b882f8b0b157..9901b95c42e8 100644 --- a/src/Deserialization.cpp +++ b/src/Deserialization.cpp @@ -27,10 +27,15 @@ class Deserializer { : external_params(external_params) { } + // Deserialize a pipeline from the given filename Pipeline deserialize(const std::string &filename); + // Deserialize a pipeline from the given input stream Pipeline deserialize(std::istream &in); + // Deserialize a pipeline from the given buffer of bytes + Pipeline deserialize(const std::vector &data); + private: // Helper function to deserialize a homogenous vector from a flatbuffer vector, // does not apply to union types like Stmt and Expr or enum types like MemoryType @@ -445,6 +450,8 @@ void Deserializer::deserialize_function(const Serialize::Func *function, Functio output_buffer = it->second; } else if (auto it = parameters_in_pipeline.find(output_buffer_name); it != parameters_in_pipeline.end()) { output_buffer = it->second; + } else if (!output_buffer_name.empty()) { + user_error << "unknown output buffer used in pipeline '" << output_buffer_name << "'\n"; } output_buffers.push_back(output_buffer); } @@ -514,6 +521,8 @@ Stmt Deserializer::deserialize_stmt(Serialize::Stmt type_code, const void *stmt) param = it->second; } else if (auto it = parameters_in_pipeline.find(param_name); it != parameters_in_pipeline.end()) { param = it->second; + } else if (!param_name.empty()) { + user_error << "unknown parameter used in pipeline '" << param_name << "'\n"; } const auto alignment = deserialize_modulus_remainder(store_stmt->alignment()); return Store::make(name, value, index, param, predicate, alignment); @@ -771,6 +780,8 @@ Expr Deserializer::deserialize_expr(Serialize::Expr type_code, const void *expr) param = it->second; } else if (auto it = parameters_in_pipeline.find(param_name); it != parameters_in_pipeline.end()) { param = it->second; + } else if (!param_name.empty()) { + user_error << "unknown parameter used in pipeline '" << param_name << "'\n"; } const auto alignment = deserialize_modulus_remainder(load_expr->alignment()); const auto type = deserialize_type(load_expr->type()); @@ -820,6 +831,8 @@ Expr Deserializer::deserialize_expr(Serialize::Expr type_code, const void *expr) param = it->second; } else if (auto it = parameters_in_pipeline.find(param_name); it != parameters_in_pipeline.end()) { param = it->second; + } else if (!param_name.empty()) { + user_error << "unknown parameter used in pipeline '" << param_name << "'\n"; } const auto type = deserialize_type(call_expr->type()); return Call::make(type, name, args, call_type, func_ptr, value_index, image, param); @@ -834,6 +847,8 @@ Expr Deserializer::deserialize_expr(Serialize::Expr type_code, const void *expr) param = it->second; } else if (auto it = parameters_in_pipeline.find(param_name); it != parameters_in_pipeline.end()) { param = it->second; + } else if (!param_name.empty()) { + user_error << "unknown parameter used in pipeline '" << param_name << "'\n"; } auto image_name = deserialize_string(variable_expr->image_name()); Buffer<> image; @@ -1031,6 +1046,8 @@ PrefetchDirective Deserializer::deserialize_prefetch_directive(const Serialize:: Parameter param; if (auto it = parameters_in_pipeline.find(param_name); it != parameters_in_pipeline.end()) { param = it->second; + } else if (!param_name.empty()) { + user_error << "unknown parameter used in pipeline '" << param_name << "'\n"; } auto hl_prefetch_directive = PrefetchDirective(); hl_prefetch_directive.name = name; @@ -1209,6 +1226,8 @@ ExternFuncArgument Deserializer::deserialize_extern_func_argument(const Serializ image_param = it->second; } else if (auto it = parameters_in_pipeline.find(image_param_name); it != parameters_in_pipeline.end()) { image_param = it->second; + } else if (!image_param_name.empty()) { + user_error << "unknown image parameter used in pipeline '" << image_param_name << "'\n"; } return ExternFuncArgument(image_param); } @@ -1304,9 +1323,12 @@ Pipeline Deserializer::deserialize(std::istream &in) { in.seekg(0, std::ios::end); int size = in.tellg(); in.seekg(0, std::ios::beg); - std::vector data(size); - in.read(data.data(), size); + std::vector data(size); + in.read((char *)data.data(), size); + return deserialize(data); +} +Pipeline Deserializer::deserialize(const std::vector &data) { const auto *pipeline_obj = Serialize::GetPipeline(data.data()); if (pipeline_obj == nullptr) { user_warning << "deserialized pipeline is empty\n"; @@ -1385,6 +1407,11 @@ Pipeline deserialize_pipeline(std::istream &in, const std::map &buffer, const std::map &external_params) { + Internal::Deserializer deserializer(external_params); + return deserializer.deserialize(buffer); +} + } // namespace Halide #else // WITH_SERIALIZATION @@ -1401,6 +1428,11 @@ Pipeline deserialize_pipeline(std::istream &in, const std::map &buffer, const std::map &external_params) { + user_error << "Deserialization is not supported in this build of Halide; try rebuilding with WITH_SERIALIZATION=ON."; + return Pipeline(); +} + } // namespace Halide #endif // WITH_SERIALIZATION diff --git a/src/Deserialization.h b/src/Deserialization.h index b4c1f7fe369d..82f7c8e7217b 100644 --- a/src/Deserialization.h +++ b/src/Deserialization.h @@ -7,16 +7,24 @@ namespace Halide { -/** - * Deserialize a Halide pipeline from a file. - * filename should always end in .hlpipe suffix. - * external_params is an optional map, all parameters in the map - * will be treated as external parameters so won't be deserialized. - */ +/// @brief Deserialize a Halide pipeline from a file. +/// @param filename The location of the file to deserialize. Must use .hlpipe extension. +/// @param external_params Map of named input/output parameters to bind with the resulting pipeline (used to avoid deserializing specific objects and enable the use of externally defined ones instead). +/// @return Returns a newly constructed deserialized Pipeline object/ Pipeline deserialize_pipeline(const std::string &filename, const std::map &external_params); +/// @brief Deserialize a Halide pipeline from an input stream. +/// @param in The input stream to read from containing a serialized Halide pipeline +/// @param external_params Map of named input/output parameters to bind with the resulting pipeline (used to avoid deserializing specific objects and enable the use of externally defined ones instead). +/// @return Returns a newly constructed deserialized Pipeline object/ Pipeline deserialize_pipeline(std::istream &in, const std::map &external_params); +/// @brief Deserialize a Halide pipeline from a byte buffer containing a serizalized pipeline in binary format +/// @param data The data buffer containing a serialized Halide pipeline +/// @param external_params Map of named input/output parameters to bind with the resulting pipeline (used to avoid deserializing specific objects and enable the use of externally defined ones instead). +/// @return Returns a newly constructed deserialized Pipeline object/ +Pipeline deserialize_pipeline(const std::vector &data, const std::map &external_params); + } // namespace Halide #endif diff --git a/src/Serialization.cpp b/src/Serialization.cpp index d9afcb5e083b..f9641919b4b2 100644 --- a/src/Serialization.cpp +++ b/src/Serialization.cpp @@ -28,8 +28,12 @@ class Serializer { public: Serializer() = default; + // Serialize the given pipeline into the given filename void serialize(const Pipeline &pipeline, const std::string &filename); + // Serialize the given pipeline into given the data buffer + void serialize(const Pipeline &pipeline, std::vector &data); + const std::map &get_external_parameters() const { return external_parameters; } @@ -1394,7 +1398,7 @@ void Serializer::build_function_mappings(const std::map & } } -void Serializer::serialize(const Pipeline &pipeline, const std::string &filename) { +void Serializer::serialize(const Pipeline &pipeline, std::vector &result) { FlatBufferBuilder builder(1024); // extract the DAG, unwrap function from Funcs @@ -1459,17 +1463,46 @@ void Serializer::serialize(const Pipeline &pipeline, const std::string &filename uint8_t *buf = builder.GetBufferPointer(); int size = builder.GetSize(); + + if (buf != nullptr && size > 0) { + result.clear(); + result.reserve(size); + result.insert(result.begin(), buf, buf + size); + } else { + user_error << "failed to serialize pipeline!\n"; + } +} + +void Serializer::serialize(const Pipeline &pipeline, const std::string &filename) { + std::vector data; + serialize(pipeline, data); std::ofstream out(filename, std::ios::out | std::ios::binary); if (!out) { user_error << "failed to open file " << filename << "\n"; exit(1); } - out.write((char *)(buf), size); + out.write((char *)(data.data()), data.size()); out.close(); } } // namespace Internal +void serialize_pipeline(const Pipeline &pipeline, std::vector &data) { + Internal::Serializer serializer; + serializer.serialize(pipeline, data); +} + +void serialize_pipeline(const Pipeline &pipeline, std::vector &data, std::map ¶ms) { + Internal::Serializer serializer; + serializer.serialize(pipeline, data); + params = serializer.get_external_parameters(); +} + +void serialize_pipeline(const Pipeline &pipeline, const std::string &filename) { + Internal::Serializer serializer; + serializer.serialize(pipeline, filename); +} + void serialize_pipeline(const Pipeline &pipeline, const std::string &filename, std::map ¶ms) { Internal::Serializer serializer; serializer.serialize(pipeline, filename); @@ -1482,6 +1515,18 @@ void serialize_pipeline(const Pipeline &pipeline, const std::string &filename, s namespace Halide { +void serialize_pipeline(const Pipeline &pipeline, std::vector &data) { + user_error << "Serialization is not supported in this build of Halide; try rebuilding with WITH_SERIALIZATION=ON."; +} + +void serialize_pipeline(const Pipeline &pipeline, std::vector &data, std::map ¶ms) { + user_error << "Serialization is not supported in this build of Halide; try rebuilding with WITH_SERIALIZATION=ON."; +} + +void serialize_pipeline(const Pipeline &pipeline, const std::string &filename) { + user_error << "Serialization is not supported in this build of Halide; try rebuilding with WITH_SERIALIZATION=ON."; +} + void serialize_pipeline(const Pipeline &pipeline, const std::string &filename, std::map ¶ms) { user_error << "Serialization is not supported in this build of Halide; try rebuilding with WITH_SERIALIZATION=ON."; } diff --git a/src/Serialization.h b/src/Serialization.h index ebbcee484147..9eb7f71c33cc 100644 --- a/src/Serialization.h +++ b/src/Serialization.h @@ -5,6 +5,27 @@ namespace Halide { +/// @brief Serialize a Halide pipeline into the given data buffer. +/// @param pipeline The Halide pipeline to serialize. +/// @param data The data buffer to store the serialized Halide pipeline into. Any existing contents will be destroyed. +/// @param params Map of named parameters which will get populated during serialization (can be used to bind external parameters to objects in the pipeline by name). +void serialize_pipeline(const Pipeline &pipeline, std::vector &data); + +/// @brief Serialize a Halide pipeline into the given data buffer. +/// @param pipeline The Halide pipeline to serialize. +/// @param data The data buffer to store the serialized Halide pipeline into. Any existing contents will be destroyed. +/// @param params Map of named parameters which will get populated during serialization (can be used to bind external parameters to objects in the pipeline by name). +void serialize_pipeline(const Pipeline &pipeline, std::vector &data, std::map ¶ms); + +/// @brief Serialize a Halide pipeline into the given filename. +/// @param pipeline The Halide pipeline to serialize. +/// @param filename The location of the file to write into to store the serialized pipeline. Any existing contents will be destroyed. +void serialize_pipeline(const Pipeline &pipeline, const std::string &filename); + +/// @brief Serialize a Halide pipeline into the given filename. +/// @param pipeline The Halide pipeline to serialize. +/// @param filename The location of the file to write into to store the serialized pipeline. Any existing contents will be destroyed. +/// @param params Map of named parameters which will get populated during serialization (can be used to bind external parameters to objects in the pipeline by name). void serialize_pipeline(const Pipeline &pipeline, const std::string &filename, std::map ¶ms); } // namespace Halide diff --git a/tutorial/CMakeLists.txt b/tutorial/CMakeLists.txt index 2221e61d74ec..862db3db6bd3 100644 --- a/tutorial/CMakeLists.txt +++ b/tutorial/CMakeLists.txt @@ -210,5 +210,6 @@ if (TARGET Halide::Mullapudi2016) set_tests_properties(tutorial_lesson_21_auto_scheduler_run PROPERTIES LABELS "tutorial;multithreaded") endif () -# Lesson 22 +# Lessons 22-23 add_tutorial(lesson_22_jit_performance.cpp) +add_tutorial(lesson_23_serialization.cpp WITH_IMAGE_IO) diff --git a/tutorial/lesson_23_serialization.cpp b/tutorial/lesson_23_serialization.cpp new file mode 100644 index 000000000000..a01de5f916fd --- /dev/null +++ b/tutorial/lesson_23_serialization.cpp @@ -0,0 +1,130 @@ +// Halide tutorial lesson 23: Serialization + +// This lesson describes how to serialize pipelines into a binary format +// which can be saved on disk, and later deserialized and loaded for +// evaluation. + +// Note that you'll need to be using a build of Halide that was configured +// using the WITH_SERIALIZATION=ON macro defined in order for this tutorial +// to work. + +// Disclaimer: Serialization is experimental in Halide 17 and is subject to +// change; we recommend that you avoid relying on it for production work at this time. + +// On linux, you can compile this tutorial and run it like so: +// g++ lesson_23*.cpp -g -I -I -L -lHalide -lpthread -ldl -o lesson_23 -std=c++17 +// LD_LIBRARY_PATH= ./lesson_23 + +// On os x: +// g++ lesson_23*.cpp -g -I -I -L -lHalide -o lesson_23 -std=c++17 +// DYLD_LIBRARY_PATH= ./lesson_23 + +// If you have the entire Halide source tree, you can also build it by +// running: +// make tutorial_lesson_23_serialization +// in a shell with the current directory at the top of the halide +// source tree. + +#include "Halide.h" +#include +#include +using namespace Halide; + +// Support code for loading pngs. +#include "halide_image_io.h" + +int main(int argc, char **argv) { + + // First we'll declare some Vars to use below. + Var x("x"), y("y"), c("c"); + + // Let's start with the same separable blur pipeline that we used in Tutorial 7, + // with the clamped boundary condition + { + // Let's create an ImageParam for an 8-bit RGB image that we'll use for input. + ImageParam input(UInt(8), 3, "input"); + + // Wrap the input in a Func that prevents reading out of bounds: + Func clamped("clamped"); + Expr clamped_x = clamp(x, 0, input.width() - 1); + Expr clamped_y = clamp(y, 0, input.height() - 1); + clamped(x, y, c) = input(clamped_x, clamped_y, c); + + // Upgrade it to 16-bit, so we can do math without it overflowing. + Func input_16("input_16"); + input_16(x, y, c) = cast(clamped(x, y, c)); + + // Blur it horizontally: + Func blur_x("blur_x"); + blur_x(x, y, c) = (input_16(x - 1, y, c) + + 2 * input_16(x, y, c) + + input_16(x + 1, y, c)) / 4; + + // Blur it vertically: + Func blur_y("blur_y"); + blur_y(x, y, c) = (blur_x(x, y - 1, c) + + 2 * blur_x(x, y, c) + + blur_x(x, y + 1, c)) / 4; + + // Convert back to 8-bit. + Func output("output"); + output(x, y, c) = cast(blur_y(x, y, c)); + + // Now lets serialize the pipeline to disk (must use the .hlpipe file extension) + Pipeline blur_pipeline(output); + std::map params; + serialize_pipeline(blur_pipeline, "blur.hlpipe", params); + + // The call to serialize_pipeline populates the params map with any input or output parameters + // that were found ... object's we'll need to attach to buffers if we wish to execute the pipeline + for(auto named_param: params) { + std::cout << "Found Param: " << named_param.first << std::endl; + } + } + + // new scope ... everything above is now destroyed! Now lets reconstruct the entire pipeline + // from scratch by deserializing it from a file + { + // Lets load a color 8-bit input and connect it to an ImageParam + Buffer rgb_image = Halide::Tools::load_image("images/rgb.png"); + ImageParam input(UInt(8), 3, "input"); + input.set(rgb_image); + + // Now lets populate the params map so we can override the input + std::map params; + params.insert({"input", input.parameter()}); + + // Lets construct a new pipeline from scratch by deserializing the file we wrote to disk + Pipeline blur_pipeline = deserialize_pipeline("blur.hlpipe", params); + + // Now realize the pipeline and blur out input image + Buffer result = blur_pipeline.realize({rgb_image.width(), rgb_image.height(), 3}); + + // Now lets save the result ... we should have another blurry parrot! + Halide::Tools::save_image(result, "another_blurry_parrot.png"); + } + + // new scope ... everything above is now destroyed! + { + // Lets do the same thing again ... construct a new pipeline from scratch by deserializing the file we wrote to disk + + // FIXME: We shouldn't have to populate the params ... but passing an empty map triggers an error in deserialize + // for a missing input param + std::map params; + ImageParam input(UInt(8), 3, "input"); + params.insert({"input", input.parameter()}); + + // Now deserialize the pipeline from file + Pipeline blur_pipeline = deserialize_pipeline("blur.hlpipe", params); + + // Now, lets serialize it to an in memory buffer ... rather than writing it to disk + std::vector data; + serialize_pipeline(blur_pipeline, data, params); + + // Now lets deserialize it from memory + Pipeline deserialized = deserialize_pipeline(data, params); + } + + printf("Success!\n"); + return 0; +}