diff --git a/404.html b/404.html index a5469f1a..4028ae4d 100644 --- a/404.html +++ b/404.html @@ -10,13 +10,13 @@ - - + +
Skip to main content

Page Not Found

We could not find what you were looking for.

Please contact the owner of the site that linked you to the original URL and let them know their link is broken.

- - + + \ No newline at end of file diff --git a/assets/js/171c4778.96672fe3.js b/assets/js/171c4778.96672fe3.js new file mode 100644 index 00000000..824f2eb0 --- /dev/null +++ b/assets/js/171c4778.96672fe3.js @@ -0,0 +1 @@ +"use strict";(self.webpackChunkdocs=self.webpackChunkdocs||[]).push([[8997],{3905:(e,t,a)=>{a.d(t,{Zo:()=>p,kt:()=>h});var r=a(7294);function o(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function n(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,r)}return a}function i(e){for(var t=1;t=0||(o[a]=e[a]);return o}(e,t);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);for(r=0;r=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(o[a]=e[a])}return o}var l=r.createContext({}),u=function(e){var t=r.useContext(l),a=t;return e&&(a="function"==typeof e?e(t):i(i({},t),e)),a},p=function(e){var t=u(e.components);return r.createElement(l.Provider,{value:t},e.children)},m="mdxType",d={inlineCode:"code",wrapper:function(e){var t=e.children;return r.createElement(r.Fragment,{},t)}},c=r.forwardRef((function(e,t){var a=e.components,o=e.mdxType,n=e.originalType,l=e.parentName,p=s(e,["components","mdxType","originalType","parentName"]),m=u(a),c=o,h=m["".concat(l,".").concat(c)]||m[c]||d[c]||n;return a?r.createElement(h,i(i({ref:t},p),{},{components:a})):r.createElement(h,i({ref:t},p))}));function h(e,t){var a=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var n=a.length,i=new Array(n);i[0]=c;var s={};for(var l in t)hasOwnProperty.call(t,l)&&(s[l]=t[l]);s.originalType=e,s[m]="string"==typeof e?e:o,i[1]=s;for(var u=2;u{a.r(t),a.d(t,{assets:()=>l,contentTitle:()=>i,default:()=>d,frontMatter:()=>n,metadata:()=>s,toc:()=>u});var r=a(7462),o=(a(7294),a(3905));const n={slug:"automerge-2",title:"Automerge 2.0",authors:["pvh"],tags:[]},i="Introducing Automerge 2.0",s={permalink:"/blog/automerge-2",editUrl:"https://github.com/automerge/automerge.github.io/edit/main/blog/2023-01-17-automerge-2/index.md",source:"@site/blog/2023-01-17-automerge-2/index.md",title:"Automerge 2.0",description:"Automerge 2.0 is here and ready for production. It\u2019s our first supported release resulting from a ground-up rewrite. The result is a production-ready CRDT with huge improvements in performance and reliability. It's available in both JavaScript and Rust, and includes TypeScript types and C bindings for use in other ecosystems. Even better, Automerge 2.0 comes with improved documentation and, for the first time, support options for production users.",date:"2023-01-17T00:00:00.000Z",formattedDate:"January 17, 2023",tags:[],readingTime:11.825,hasTruncateMarker:!1,authors:[{name:"PVH",title:"Contributor",url:"https://github.com/pvh",key:"pvh"}],frontMatter:{slug:"automerge-2",title:"Automerge 2.0",authors:["pvh"],tags:[]},prevItem:{title:'Automerge-Repo: A "batteries-included" toolkit for building local-first applications',permalink:"/blog/2023/11/06/automerge-repo"},nextItem:{title:"Welcome",permalink:"/blog/welcome"}},l={authorsImageUrls:[void 0]},u=[{value:"Automerge, CRDTs, and Local-first Software",id:"automerge-crdts-and-local-first-software",level:2},{value:"Automerge-RS: Rebuilt for Performance & Portability",id:"automerge-rs-rebuilt-for-performance--portability",level:2},{value:"Documenting Automerge",id:"documenting-automerge",level:2},{value:"Supporting Automerge",id:"supporting-automerge",level:2},{value:"Performance: Speed, Memory and Disk",id:"performance-speed-memory-and-disk",level:2},{value:"Portability & Mobile Devices",id:"portability--mobile-devices",level:2},{value:"Native Rich Text Support",id:"native-rich-text-support",level:2},{value:"Automerge-Repo",id:"automerge-repo",level:2},{value:"Rust Developer Experience Improvements",id:"rust-developer-experience-improvements",level:2},{value:"Improved Synchronization",id:"improved-synchronization",level:2},{value:"Built-in Branches",id:"built-in-branches",level:2},{value:"History Management",id:"history-management",level:2}],p={toc:u},m="wrapper";function d(e){let{components:t,...a}=e;return(0,o.kt)(m,(0,r.Z)({},p,a,{components:t,mdxType:"MDXLayout"}),(0,o.kt)("p",null,"Automerge 2.0 is here and ready for production. It\u2019s our first supported release resulting from a ground-up rewrite. The result is a production-ready CRDT with huge improvements in performance and reliability. It's available in both JavaScript and Rust, and includes TypeScript types and C bindings for use in other ecosystems. Even better, Automerge 2.0 comes with improved documentation and, for the first time, support options for production users."),(0,o.kt)("h2",{id:"automerge-crdts-and-local-first-software"},"Automerge, CRDTs, and Local-first Software"),(0,o.kt)("p",null,"Before getting into the details of why we're excited about Automerge 2.0, let's take a bit of time to explain what Automerge is for anyone unfamiliar with the project."),(0,o.kt)("p",null,"Automerge is a ",(0,o.kt)("a",{parentName:"p",href:"https://crdt.tech/"},"CRDT"),', or "conflict-free replicated data type", but if you\'re allergic to buzzwords you can just think of it as a version controlled data structure. Automerge lets you record changes made to data and then replay them in other places, reliably producing the same result in each. It supports JSON-like data, including arbitrarily nested maps and arrays, as well as some more advanced data types such as text and numeric counters.'),(0,o.kt)("p",null,"This is useful for quite a few reasons: you can use it to implement real-time collaboration for an application without having to figure out tricky application-specific algorithms on the server. You can also use it to better support offline work. We think it has even more potential than just that."),(0,o.kt)("p",null,"Since the rise of the cloud, developers have largely had to choose between building cloud software or traditional installed software. Although installed software has some reliability and performance benefits, cloud software has dominated the market. Cloud software makes sharing data between users easy and includes ubiquitous access from any computing device. Unfortunately, the advantages of cloud software come at a high price. Cloud software is fragile and prone to outages, rarely supports offline use, and is expensive to scale to large audiences."),(0,o.kt)("p",null,"At Ink & Switch, we\u2019ve been researching a model for developing software which we call ",(0,o.kt)("a",{parentName:"p",href:"https://www.inkandswitch.com/local-first/"},"local-first software"),", with the goal of combining the best of both worlds: reliable, locally-executed software paired with scalable offline-friendly collaboration infrastructure. We believe that a strong data model based on recording change over time for every application should be a cornerstone of that effort."),(0,o.kt)("h2",{id:"automerge-rs-rebuilt-for-performance--portability"},"Automerge-RS: Rebuilt for Performance & Portability"),(0,o.kt)("p",null,"Earlier versions of Automerge were implemented in pure JavaScript. Our initial implementations were theoretically sound but much too slow and used too much memory for most production use cases."),(0,o.kt)("p",null,"Furthermore, JavaScript support on mobile devices and embedded systems is limited. We wanted a fast and efficient version of Automerge that was available everywhere: in the browser, on any mobile device, and even microcontrollers like the ",(0,o.kt)("a",{parentName:"p",href:"https://en.wikipedia.org/wiki/ESP32"},"ESP32"),"."),(0,o.kt)("p",null,"Instead of trying to coordinate multiple distinct versions of Automerge, we decided to rewrite Automerge in Rust and use platform-specific wrappers to make it available in each language ecosystem. This way we can be confident that the core CRDT logic is identical across all platforms and that everyone benefits from new features and optimizations together."),(0,o.kt)("p",null,"For JavaScript applications, this means compiling the Rust to WebAssembly and providing a JavaScript wrapper that maintains the existing Automerge API. Rust applications can obviously use the library directly, and we're making sure that it's as easy as possible to implement support in other languages with well-designed traits and a comprehensive set of C bindings."),(0,o.kt)("p",null,"To deliver this new version, lab members Alex Good and Orion Henry teamed up with open source collaborators including Andrew Jeffery and Jason Kankiewicz to polish and optimize the Rust implementation and JavaScript wrapper. The result is a codebase that is hundreds of times faster than past releases, radically more memory efficient, better tested, and more reliable."),(0,o.kt)("h2",{id:"documenting-automerge"},"Documenting Automerge"),(0,o.kt)("p",null,"With Automerge 2.0 we've made a big investment in improving documentation. In addition to ",(0,o.kt)("a",{parentName:"p",href:"https://github.com/automerge/automerge-rs/tree/main/javascript/examples"},"sample code"),", we now have a ",(0,o.kt)("a",{parentName:"p",href:"https://automerge.org/docs/tutorial/introduction/"},"tutorial")," and ",(0,o.kt)("a",{parentName:"p",href:"https://automerge.org/docs/quickstart/"},"quick-start guide")," that support both Vite and create-react-app, as well as ",(0,o.kt)("a",{parentName:"p",href:"https://automerge.org/docs/how-it-works/backend/"},"internals")," documentation, ",(0,o.kt)("a",{parentName:"p",href:"https://alexjg.github.io/automerge-storage-docs/"},"file format")," and ",(0,o.kt)("a",{parentName:"p",href:"https://automerge.org/docs/how-it-works/sync/"},"sync protocol")," documentation. This work was led by lab alumnus Rae McKelvey and we hope it helps make getting started with Automerge much easier. Please let us know if there are other topics or areas you'd like to see covered!"),(0,o.kt)("h2",{id:"supporting-automerge"},"Supporting Automerge"),(0,o.kt)("p",null,"Those who have been following Automerge for a while may have noticed that we describe Automerge 2.0 as our first ",(0,o.kt)("em",{parentName:"p"},"supported")," release. That\u2019s because as part of the Automerge 2.0 release we\u2019ve brought Alex Good onto the team full-time to provide support to external users, handle documentation, release management, and\u2014of course\u2014to continue implementing new Automerge features for the community."),(0,o.kt)("p",null,"This is a big moment for Ink & Switch and the Automerge project: we\u2019re now able to provide support to our users thanks to sponsorship from enterprises like ",(0,o.kt)("a",{parentName:"p",href:"https://fly.io/"},"Fly.io"),", ",(0,o.kt)("a",{parentName:"p",href:"https://www.prisma.io/"},"Prisma"),", and ",(0,o.kt)("a",{parentName:"p",href:"https://bowtie.works/"},"Bowtie")," as well as so many others who have contributed either directly to Automerge or through supporting Martin Kleppmann on Patreon."),(0,o.kt)("p",null,"If your business is interested in sponsoring Automerge, you can ",(0,o.kt)("a",{parentName:"p",href:"https://github.com/sponsors/automerge"},"sponsor us directly"),", or ",(0,o.kt)("a",{parentName:"p",href:"mailto:hello@inkandswitch.com"},"get in touch with us for more information or other sponsorship methods"),". Every little bit helps, and the more sponsors we have, the more work we can do while still remaining an independent open source project."),(0,o.kt)("blockquote",null,(0,o.kt)("p",{parentName:"blockquote"},"At Bowtie we support Automerge because it's the best way to achieve the resilliency properties that we're delivering to globally distributed private networks. It's clear to me that our sponsorship has furthered our software, and that this crew are among the best distributed-systems thinkers in the business.\n-- Issac Kelly, CTO, Bowtie.")),(0,o.kt)("h2",{id:"performance-speed-memory-and-disk"},"Performance: Speed, Memory and Disk"),(0,o.kt)("p",null,"Using a CRDT inherently comes with overhead: we have to track additional information in order to be able to correctly merge work from different sources. The goal of all CRDT authors is to find the right trade-offs between preserving useful history, reducing CPU overhead, and efficiently storing data in memory and on disk."),(0,o.kt)("p",null,"With the Automerge project, our goal is to retain the full history of any document and allow an author to reconstruct any point in time on demand. As software developers we're accustomed to having this power: it's hard to imagine version control without history."),(0,o.kt)("p",null,"With Automerge 2.0, we've brought together an efficient binary data format with fast updates, save, and load performance. Without getting too into the details, we accomplish this by packing data efficiently in memory, ensuring that related data is stored close together for quick retrieval."),(0,o.kt)("p",null,"Let's take a look at some numbers. One of the most challenging benchmarks for CRDTs is realtime text collaboration. That's because a long editing session can result in hundreds of thousands of individual keystrokes to record and synchronize. Martin Kleppmann recorded the keystrokes that went into writing an academic paper and replaying that data has become a ",(0,o.kt)("a",{parentName:"p",href:"https://github.com/automerge/automerge-perf"},"popular benchmark")," for CRDTs."),(0,o.kt)("table",null,(0,o.kt)("thead",{parentName:"table"},(0,o.kt)("tr",{parentName:"thead"},(0,o.kt)("th",{parentName:"tr",align:null},"Insert ~260k operations"),(0,o.kt)("th",{parentName:"tr",align:null},"Timing (ms)"),(0,o.kt)("th",{parentName:"tr",align:null},"Memory (bytes)"))),(0,o.kt)("tbody",{parentName:"table"},(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"Automerge 0.14"),(0,o.kt)("td",{parentName:"tr",align:null},"~500,000"),(0,o.kt)("td",{parentName:"tr",align:null},"~1,100,000,000")),(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"Automerge 1.0.1"),(0,o.kt)("td",{parentName:"tr",align:null},"13,052"),(0,o.kt)("td",{parentName:"tr",align:null},"184,721,408")),(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"Automerge 2.0.1"),(0,o.kt)("td",{parentName:"tr",align:null},"1,816"),(0,o.kt)("td",{parentName:"tr",align:null},"44,523,520")),(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"Yjs"),(0,o.kt)("td",{parentName:"tr",align:null},"1,074"),(0,o.kt)("td",{parentName:"tr",align:null},"10,141,696")),(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"Automerge 2.0.2-unstable"),(0,o.kt)("td",{parentName:"tr",align:null},"661"),(0,o.kt)("td",{parentName:"tr",align:null},"22,953,984")))),(0,o.kt)("p",null,"Of course, even the most productive authors struggle to type an entire paper quite so quickly. Indeed, writing a paper can occur over months or even years, making both storage size on disk and load performance important as well."),(0,o.kt)("table",null,(0,o.kt)("thead",{parentName:"table"},(0,o.kt)("tr",{parentName:"thead"},(0,o.kt)("th",{parentName:"tr",align:null},"Size on Disk"),(0,o.kt)("th",{parentName:"tr",align:null},"bytes"))),(0,o.kt)("tbody",{parentName:"table"},(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"plain text"),(0,o.kt)("td",{parentName:"tr",align:null},"107,121")),(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"automerge 2.0"),(0,o.kt)("td",{parentName:"tr",align:null},"129,062")),(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"automerge 0.14"),(0,o.kt)("td",{parentName:"tr",align:null},"146,406,415")))),(0,o.kt)("p",null,"The binary format works wonders in this example, encoding a full history for the document with only 30% overhead. That's less than one additional byte per character! The naive JSON encoding often used circa automerge 0.14 could exceed 1,300 bytes ",(0,o.kt)("em",{parentName:"p"},"per character"),". If you'd like to learn more about the file format, we have a ",(0,o.kt)("a",{parentName:"p",href:"https://alexjg.github.io/automerge-storage-docs/"},"specification")," document."),(0,o.kt)("table",null,(0,o.kt)("thead",{parentName:"table"},(0,o.kt)("tr",{parentName:"thead"},(0,o.kt)("th",{parentName:"tr",align:null},"Load ~260k operations"),(0,o.kt)("th",{parentName:"tr",align:null},"Timing (ms)"))),(0,o.kt)("tbody",{parentName:"table"},(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"Automerge 1.0.1"),(0,o.kt)("td",{parentName:"tr",align:null},"590")),(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"Automerge 2.0.1"),(0,o.kt)("td",{parentName:"tr",align:null},"593")),(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"Automerge 2.0.2-unstable"),(0,o.kt)("td",{parentName:"tr",align:null},"438")))),(0,o.kt)("p",null,"Loading the compressed document is fast as well, ensuring the best possible start-up time."),(0,o.kt)("p",null,"While we are proud of these results, we will continue to invest in improved performance with each release as you can see with the preliminary numbers for the upcoming Automerge 2.0.2 release."),(0,o.kt)("p",null,"A few notes about methodology before we move on. The particular implementation we used to run the benchmarks can be found ",(0,o.kt)("a",{parentName:"p",href:"https://github.com/alexjg/automerge-perf-comparisons"},"here"),'. These numbers were produced on Ryzen 9 7900X. The "timing" column is how long it takes to apply every single edit in the trace, whilst the "memory" common is the peak memory usage during this process.'),(0,o.kt)("p",null,'The improvements found in "2.0.2-unstable" mostly result from an upcoming improved API for text. Also note that the "automerge 1.0.1" here is actually the ',(0,o.kt)("inlineCode",{parentName:"p"},"automerge@1.0.1-preview-7")," release. Automerge 1.0.1 was a significant rewrite from 0.14 and has a similar architecture to the Rust implementation. Improvements between 1.0.1 and 2.0.1 are a result of both optimization and adopting WebAssembly rather than an architectural change."),(0,o.kt)("h2",{id:"portability--mobile-devices"},"Portability & Mobile Devices"),(0,o.kt)("p",null,"Because the core logic of Automerge is now built in Rust, we're able to port it more easily to a wide variety of environments and bind it to almost any language. We have users today who directly build on Automerge using the Rust APIs (and the helpful ",(0,o.kt)("a",{parentName:"p",href:"https://github.com/automerge/autosurgeon"},"autosurgeon")," library). We also have a ",(0,o.kt)("a",{parentName:"p",href:"https://github.com/automerge/automerge-rs/tree/main/rust/automerge-c"},"C-bindings API")," designed and contributed by Jason Kankiewicz, and are excited to see the ",(0,o.kt)("a",{parentName:"p",href:"https://github.com/automerge/automerge-go"},(0,o.kt)("inlineCode",{parentName:"a"},"automerge-go"))," implementation underway by Conrad Irwin."),(0,o.kt)("p",null,"In the future, we hope to provide bindings for other languages including Swift, Kotlin, and Python. If you're interested in getting involved in those projects please let us know!"),(0,o.kt)("p",null,"One important note is that React-Native does not support WASM today. Developers building mobile applications will need to bind directly via C. If you're interested in either working on or sponsoring work on this problem, feel free to get in touch."),(0,o.kt)("h1",{id:"whats-next"},"What\u2019s Next"),(0,o.kt)("p",null,"With the release of Automerge 2.0 out the door, we will of course be listening closely to the community about their experience with the release, but in the months ahead, we expect to work on at least some of the following features:"),(0,o.kt)("h2",{id:"native-rich-text-support"},"Native Rich Text Support"),(0,o.kt)("p",null,"As with most CRDTs, Automerge originally focused on optimizing editing of plaintext. In the ",(0,o.kt)("a",{parentName:"p",href:"https://www.inkandswitch.com/peritext/"},"Peritext paper")," by Ink & Switch we discuss an algorithm for supporting rich text with good merging accuracy, and we are planning to integrate this algorithm into Automerge. Support for rich text will also make it easier to implement features like comments or cursor and selection sharing."),(0,o.kt)("h2",{id:"automerge-repo"},"Automerge-Repo"),(0,o.kt)("p",null,"We\u2019ve worked hard to keep Automerge platform-agnostic and support a wide variety of deployment environments. We don\u2019t require a particular network stack or storage system, and Automerge has been used successfully in, client-server web applications, peer-to-peer desktop software, and as a data synchronization engine for cloud services. Unfortunately, excluding network and storage from the library has left a lot of the busy-work up to application developers, and asked them to learn a lot about distributed systems just to get started."),(0,o.kt)("p",null,"Our new library, ",(0,o.kt)("a",{parentName:"p",href:"https://github.com/automerge/automerge-repo"},"Automerge-Repo"),", is a modular batteries-included approach to building web applications with Automerge. It works both in the browser (desktop and mobile) and in Node, and supports a variety of networking and storage adapters. There are even text editor bindings for Quill and Prosemirror as well as React Hooks to make it easy to get started quickly."),(0,o.kt)("p",null,"It's under active development, and available in beta right now. We'll talk more about it when we announce GA, but if you're starting a browser-based application now, it's probably the right place to start."),(0,o.kt)("h2",{id:"rust-developer-experience-improvements"},"Rust Developer Experience Improvements"),(0,o.kt)("p",null,"We've seen tremendous enthusiasm for the native Rust experience of Automerge, and the current Rust API is powerful and fast. Unfortunately, it's also low-level and can be difficult to work with directly. To make building Rust applications against automerge easier, Alex built ",(0,o.kt)("a",{parentName:"p",href:"https://github.com/automerge/autosurgeon"},"Autosurgeon"),", a library that helps bind Rust data structures to Automerge documents, and we'll continue to listen to our Rust users and improve on that experience."),(0,o.kt)("h2",{id:"improved-synchronization"},"Improved Synchronization"),(0,o.kt)("p",null,"Automerge's current synchronization system has some great properties. In many cases it can bring two clients up to date with only a single round-trip each direction. That said, we see big potential to improve the CPU performance of this process, and also lots of opportunity to improve sync performance of many documents at once. We also expect to provide other optimizations our users and sponsors have requested, such as more efficient first-document loading, network compaction of related changes, and enabling something akin to a Git \u201cshallow clone\u201d for clients which don't need historical data."),(0,o.kt)("h2",{id:"built-in-branches"},"Built-in Branches"),(0,o.kt)("p",null,"While we retain the full history of Automerge documents and provide APIs to access it, we don\u2019t currently provide an efficient way to reconcile many closely related versions of a given document. This feature is particularly valuable for supporting offline collaboration in professional environments and (combined with Rich Text Support) should make it much easier for our friends in journalism organizations to build powerful and accurate editing tools."),(0,o.kt)("h2",{id:"history-management"},"History Management"),(0,o.kt)("p",null,"Today the best way to remove something from an Automerge document's history is to recreate the document from scratch or to reset to a time before that change went in. In the future, we plan to provide additional tools to give developers more control over document history. We expect this to include the ability to share just the latest version of a document (similar to a shallow clone in ",(0,o.kt)("inlineCode",{parentName:"p"},"git"),"), and to share updates that bypass changes you don't want to share (as when a developer squashes commits before publishing). "),(0,o.kt)("h1",{id:"conclusion"},"Conclusion"),(0,o.kt)("p",null,"Automerge 2.0 is here, it\u2019s ready for you, and we\u2019re tremendously excited to share it with you. We\u2019ve made Automerge faster, more memory efficient, and we\u2019re bringing it to more platforms than ever. We\u2019re adding features, making it easier to adopt, and have begun growing a team to support it. There has never been a better moment to start building local-first software: why not ",(0,o.kt)("a",{parentName:"p",href:"https://automerge.org/docs/hello/"},"give it a try"),", and please feel welcome to ",(0,o.kt)("a",{parentName:"p",href:"https://join.slack.com/t/automerge/shared_invite/zt-e4p3760n-kKh7r3KRH1YwwNfiZM8ktw"},"join us in the Automerge Slack"),", too."),(0,o.kt)("admonition",{type:"caution"},(0,o.kt)("p",{parentName:"admonition"},"A note to existing users: Automerge 2.0 is found on npm at ",(0,o.kt)("inlineCode",{parentName:"p"},"@automerge/automerge"),". We have deprecated the ",(0,o.kt)("inlineCode",{parentName:"p"},"automerge")," package.")))}d.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/171c4778.fa1e6097.js b/assets/js/171c4778.fa1e6097.js deleted file mode 100644 index a751b405..00000000 --- a/assets/js/171c4778.fa1e6097.js +++ /dev/null @@ -1 +0,0 @@ -"use strict";(self.webpackChunkdocs=self.webpackChunkdocs||[]).push([[8997],{3905:(e,t,a)=>{a.d(t,{Zo:()=>p,kt:()=>h});var r=a(7294);function o(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function n(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,r)}return a}function i(e){for(var t=1;t=0||(o[a]=e[a]);return o}(e,t);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);for(r=0;r=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(o[a]=e[a])}return o}var l=r.createContext({}),u=function(e){var t=r.useContext(l),a=t;return e&&(a="function"==typeof e?e(t):i(i({},t),e)),a},p=function(e){var t=u(e.components);return r.createElement(l.Provider,{value:t},e.children)},m="mdxType",d={inlineCode:"code",wrapper:function(e){var t=e.children;return r.createElement(r.Fragment,{},t)}},c=r.forwardRef((function(e,t){var a=e.components,o=e.mdxType,n=e.originalType,l=e.parentName,p=s(e,["components","mdxType","originalType","parentName"]),m=u(a),c=o,h=m["".concat(l,".").concat(c)]||m[c]||d[c]||n;return a?r.createElement(h,i(i({ref:t},p),{},{components:a})):r.createElement(h,i({ref:t},p))}));function h(e,t){var a=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var n=a.length,i=new Array(n);i[0]=c;var s={};for(var l in t)hasOwnProperty.call(t,l)&&(s[l]=t[l]);s.originalType=e,s[m]="string"==typeof e?e:o,i[1]=s;for(var u=2;u{a.r(t),a.d(t,{assets:()=>l,contentTitle:()=>i,default:()=>d,frontMatter:()=>n,metadata:()=>s,toc:()=>u});var r=a(7462),o=(a(7294),a(3905));const n={slug:"automerge-2",title:"Automerge 2.0",authors:["pvh"],tags:[]},i="Introducing Automerge 2.0",s={permalink:"/blog/automerge-2",editUrl:"https://github.com/automerge/automerge.github.io/edit/main/blog/2023-01-17-automerge-2/index.md",source:"@site/blog/2023-01-17-automerge-2/index.md",title:"Automerge 2.0",description:"Automerge 2.0 is here and ready for production. It\u2019s our first supported release resulting from a ground-up rewrite. The result is a production-ready CRDT with huge improvements in performance and reliability. It's available in both JavaScript and Rust, and includes TypeScript types and C bindings for use in other ecosystems. Even better, Automerge 2.0 comes with improved documentation and, for the first time, support options for production users.",date:"2023-01-17T00:00:00.000Z",formattedDate:"January 17, 2023",tags:[],readingTime:11.825,hasTruncateMarker:!1,authors:[{name:"PVH",title:"Contributor",url:"https://github.com/pvh",key:"pvh"}],frontMatter:{slug:"automerge-2",title:"Automerge 2.0",authors:["pvh"],tags:[]},nextItem:{title:"Welcome",permalink:"/blog/welcome"}},l={authorsImageUrls:[void 0]},u=[{value:"Automerge, CRDTs, and Local-first Software",id:"automerge-crdts-and-local-first-software",level:2},{value:"Automerge-RS: Rebuilt for Performance & Portability",id:"automerge-rs-rebuilt-for-performance--portability",level:2},{value:"Documenting Automerge",id:"documenting-automerge",level:2},{value:"Supporting Automerge",id:"supporting-automerge",level:2},{value:"Performance: Speed, Memory and Disk",id:"performance-speed-memory-and-disk",level:2},{value:"Portability & Mobile Devices",id:"portability--mobile-devices",level:2},{value:"Native Rich Text Support",id:"native-rich-text-support",level:2},{value:"Automerge-Repo",id:"automerge-repo",level:2},{value:"Rust Developer Experience Improvements",id:"rust-developer-experience-improvements",level:2},{value:"Improved Synchronization",id:"improved-synchronization",level:2},{value:"Built-in Branches",id:"built-in-branches",level:2},{value:"History Management",id:"history-management",level:2}],p={toc:u},m="wrapper";function d(e){let{components:t,...a}=e;return(0,o.kt)(m,(0,r.Z)({},p,a,{components:t,mdxType:"MDXLayout"}),(0,o.kt)("p",null,"Automerge 2.0 is here and ready for production. It\u2019s our first supported release resulting from a ground-up rewrite. The result is a production-ready CRDT with huge improvements in performance and reliability. It's available in both JavaScript and Rust, and includes TypeScript types and C bindings for use in other ecosystems. Even better, Automerge 2.0 comes with improved documentation and, for the first time, support options for production users."),(0,o.kt)("h2",{id:"automerge-crdts-and-local-first-software"},"Automerge, CRDTs, and Local-first Software"),(0,o.kt)("p",null,"Before getting into the details of why we're excited about Automerge 2.0, let's take a bit of time to explain what Automerge is for anyone unfamiliar with the project."),(0,o.kt)("p",null,"Automerge is a ",(0,o.kt)("a",{parentName:"p",href:"https://crdt.tech/"},"CRDT"),', or "conflict-free replicated data type", but if you\'re allergic to buzzwords you can just think of it as a version controlled data structure. Automerge lets you record changes made to data and then replay them in other places, reliably producing the same result in each. It supports JSON-like data, including arbitrarily nested maps and arrays, as well as some more advanced data types such as text and numeric counters.'),(0,o.kt)("p",null,"This is useful for quite a few reasons: you can use it to implement real-time collaboration for an application without having to figure out tricky application-specific algorithms on the server. You can also use it to better support offline work. We think it has even more potential than just that."),(0,o.kt)("p",null,"Since the rise of the cloud, developers have largely had to choose between building cloud software or traditional installed software. Although installed software has some reliability and performance benefits, cloud software has dominated the market. Cloud software makes sharing data between users easy and includes ubiquitous access from any computing device. Unfortunately, the advantages of cloud software come at a high price. Cloud software is fragile and prone to outages, rarely supports offline use, and is expensive to scale to large audiences."),(0,o.kt)("p",null,"At Ink & Switch, we\u2019ve been researching a model for developing software which we call ",(0,o.kt)("a",{parentName:"p",href:"https://www.inkandswitch.com/local-first/"},"local-first software"),", with the goal of combining the best of both worlds: reliable, locally-executed software paired with scalable offline-friendly collaboration infrastructure. We believe that a strong data model based on recording change over time for every application should be a cornerstone of that effort."),(0,o.kt)("h2",{id:"automerge-rs-rebuilt-for-performance--portability"},"Automerge-RS: Rebuilt for Performance & Portability"),(0,o.kt)("p",null,"Earlier versions of Automerge were implemented in pure JavaScript. Our initial implementations were theoretically sound but much too slow and used too much memory for most production use cases."),(0,o.kt)("p",null,"Furthermore, JavaScript support on mobile devices and embedded systems is limited. We wanted a fast and efficient version of Automerge that was available everywhere: in the browser, on any mobile device, and even microcontrollers like the ",(0,o.kt)("a",{parentName:"p",href:"https://en.wikipedia.org/wiki/ESP32"},"ESP32"),"."),(0,o.kt)("p",null,"Instead of trying to coordinate multiple distinct versions of Automerge, we decided to rewrite Automerge in Rust and use platform-specific wrappers to make it available in each language ecosystem. This way we can be confident that the core CRDT logic is identical across all platforms and that everyone benefits from new features and optimizations together."),(0,o.kt)("p",null,"For JavaScript applications, this means compiling the Rust to WebAssembly and providing a JavaScript wrapper that maintains the existing Automerge API. Rust applications can obviously use the library directly, and we're making sure that it's as easy as possible to implement support in other languages with well-designed traits and a comprehensive set of C bindings."),(0,o.kt)("p",null,"To deliver this new version, lab members Alex Good and Orion Henry teamed up with open source collaborators including Andrew Jeffery and Jason Kankiewicz to polish and optimize the Rust implementation and JavaScript wrapper. The result is a codebase that is hundreds of times faster than past releases, radically more memory efficient, better tested, and more reliable."),(0,o.kt)("h2",{id:"documenting-automerge"},"Documenting Automerge"),(0,o.kt)("p",null,"With Automerge 2.0 we've made a big investment in improving documentation. In addition to ",(0,o.kt)("a",{parentName:"p",href:"https://github.com/automerge/automerge-rs/tree/main/javascript/examples"},"sample code"),", we now have a ",(0,o.kt)("a",{parentName:"p",href:"https://automerge.org/docs/tutorial/introduction/"},"tutorial")," and ",(0,o.kt)("a",{parentName:"p",href:"https://automerge.org/docs/quickstart/"},"quick-start guide")," that support both Vite and create-react-app, as well as ",(0,o.kt)("a",{parentName:"p",href:"https://automerge.org/docs/how-it-works/backend/"},"internals")," documentation, ",(0,o.kt)("a",{parentName:"p",href:"https://alexjg.github.io/automerge-storage-docs/"},"file format")," and ",(0,o.kt)("a",{parentName:"p",href:"https://automerge.org/docs/how-it-works/sync/"},"sync protocol")," documentation. This work was led by lab alumnus Rae McKelvey and we hope it helps make getting started with Automerge much easier. Please let us know if there are other topics or areas you'd like to see covered!"),(0,o.kt)("h2",{id:"supporting-automerge"},"Supporting Automerge"),(0,o.kt)("p",null,"Those who have been following Automerge for a while may have noticed that we describe Automerge 2.0 as our first ",(0,o.kt)("em",{parentName:"p"},"supported")," release. That\u2019s because as part of the Automerge 2.0 release we\u2019ve brought Alex Good onto the team full-time to provide support to external users, handle documentation, release management, and\u2014of course\u2014to continue implementing new Automerge features for the community."),(0,o.kt)("p",null,"This is a big moment for Ink & Switch and the Automerge project: we\u2019re now able to provide support to our users thanks to sponsorship from enterprises like ",(0,o.kt)("a",{parentName:"p",href:"https://fly.io/"},"Fly.io"),", ",(0,o.kt)("a",{parentName:"p",href:"https://www.prisma.io/"},"Prisma"),", and ",(0,o.kt)("a",{parentName:"p",href:"https://bowtie.works/"},"Bowtie")," as well as so many others who have contributed either directly to Automerge or through supporting Martin Kleppmann on Patreon."),(0,o.kt)("p",null,"If your business is interested in sponsoring Automerge, you can ",(0,o.kt)("a",{parentName:"p",href:"https://github.com/sponsors/automerge"},"sponsor us directly"),", or ",(0,o.kt)("a",{parentName:"p",href:"mailto:hello@inkandswitch.com"},"get in touch with us for more information or other sponsorship methods"),". Every little bit helps, and the more sponsors we have, the more work we can do while still remaining an independent open source project."),(0,o.kt)("blockquote",null,(0,o.kt)("p",{parentName:"blockquote"},"At Bowtie we support Automerge because it's the best way to achieve the resilliency properties that we're delivering to globally distributed private networks. It's clear to me that our sponsorship has furthered our software, and that this crew are among the best distributed-systems thinkers in the business.\n-- Issac Kelly, CTO, Bowtie.")),(0,o.kt)("h2",{id:"performance-speed-memory-and-disk"},"Performance: Speed, Memory and Disk"),(0,o.kt)("p",null,"Using a CRDT inherently comes with overhead: we have to track additional information in order to be able to correctly merge work from different sources. The goal of all CRDT authors is to find the right trade-offs between preserving useful history, reducing CPU overhead, and efficiently storing data in memory and on disk."),(0,o.kt)("p",null,"With the Automerge project, our goal is to retain the full history of any document and allow an author to reconstruct any point in time on demand. As software developers we're accustomed to having this power: it's hard to imagine version control without history."),(0,o.kt)("p",null,"With Automerge 2.0, we've brought together an efficient binary data format with fast updates, save, and load performance. Without getting too into the details, we accomplish this by packing data efficiently in memory, ensuring that related data is stored close together for quick retrieval."),(0,o.kt)("p",null,"Let's take a look at some numbers. One of the most challenging benchmarks for CRDTs is realtime text collaboration. That's because a long editing session can result in hundreds of thousands of individual keystrokes to record and synchronize. Martin Kleppmann recorded the keystrokes that went into writing an academic paper and replaying that data has become a ",(0,o.kt)("a",{parentName:"p",href:"https://github.com/automerge/automerge-perf"},"popular benchmark")," for CRDTs."),(0,o.kt)("table",null,(0,o.kt)("thead",{parentName:"table"},(0,o.kt)("tr",{parentName:"thead"},(0,o.kt)("th",{parentName:"tr",align:null},"Insert ~260k operations"),(0,o.kt)("th",{parentName:"tr",align:null},"Timing (ms)"),(0,o.kt)("th",{parentName:"tr",align:null},"Memory (bytes)"))),(0,o.kt)("tbody",{parentName:"table"},(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"Automerge 0.14"),(0,o.kt)("td",{parentName:"tr",align:null},"~500,000"),(0,o.kt)("td",{parentName:"tr",align:null},"~1,100,000,000")),(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"Automerge 1.0.1"),(0,o.kt)("td",{parentName:"tr",align:null},"13,052"),(0,o.kt)("td",{parentName:"tr",align:null},"184,721,408")),(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"Automerge 2.0.1"),(0,o.kt)("td",{parentName:"tr",align:null},"1,816"),(0,o.kt)("td",{parentName:"tr",align:null},"44,523,520")),(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"Yjs"),(0,o.kt)("td",{parentName:"tr",align:null},"1,074"),(0,o.kt)("td",{parentName:"tr",align:null},"10,141,696")),(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"Automerge 2.0.2-unstable"),(0,o.kt)("td",{parentName:"tr",align:null},"661"),(0,o.kt)("td",{parentName:"tr",align:null},"22,953,984")))),(0,o.kt)("p",null,"Of course, even the most productive authors struggle to type an entire paper quite so quickly. Indeed, writing a paper can occur over months or even years, making both storage size on disk and load performance important as well."),(0,o.kt)("table",null,(0,o.kt)("thead",{parentName:"table"},(0,o.kt)("tr",{parentName:"thead"},(0,o.kt)("th",{parentName:"tr",align:null},"Size on Disk"),(0,o.kt)("th",{parentName:"tr",align:null},"bytes"))),(0,o.kt)("tbody",{parentName:"table"},(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"plain text"),(0,o.kt)("td",{parentName:"tr",align:null},"107,121")),(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"automerge 2.0"),(0,o.kt)("td",{parentName:"tr",align:null},"129,062")),(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"automerge 0.14"),(0,o.kt)("td",{parentName:"tr",align:null},"146,406,415")))),(0,o.kt)("p",null,"The binary format works wonders in this example, encoding a full history for the document with only 30% overhead. That's less than one additional byte per character! The naive JSON encoding often used circa automerge 0.14 could exceed 1,300 bytes ",(0,o.kt)("em",{parentName:"p"},"per character"),". If you'd like to learn more about the file format, we have a ",(0,o.kt)("a",{parentName:"p",href:"https://alexjg.github.io/automerge-storage-docs/"},"specification")," document."),(0,o.kt)("table",null,(0,o.kt)("thead",{parentName:"table"},(0,o.kt)("tr",{parentName:"thead"},(0,o.kt)("th",{parentName:"tr",align:null},"Load ~260k operations"),(0,o.kt)("th",{parentName:"tr",align:null},"Timing (ms)"))),(0,o.kt)("tbody",{parentName:"table"},(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"Automerge 1.0.1"),(0,o.kt)("td",{parentName:"tr",align:null},"590")),(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"Automerge 2.0.1"),(0,o.kt)("td",{parentName:"tr",align:null},"593")),(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"Automerge 2.0.2-unstable"),(0,o.kt)("td",{parentName:"tr",align:null},"438")))),(0,o.kt)("p",null,"Loading the compressed document is fast as well, ensuring the best possible start-up time."),(0,o.kt)("p",null,"While we are proud of these results, we will continue to invest in improved performance with each release as you can see with the preliminary numbers for the upcoming Automerge 2.0.2 release."),(0,o.kt)("p",null,"A few notes about methodology before we move on. The particular implementation we used to run the benchmarks can be found ",(0,o.kt)("a",{parentName:"p",href:"https://github.com/alexjg/automerge-perf-comparisons"},"here"),'. These numbers were produced on Ryzen 9 7900X. The "timing" column is how long it takes to apply every single edit in the trace, whilst the "memory" common is the peak memory usage during this process.'),(0,o.kt)("p",null,'The improvements found in "2.0.2-unstable" mostly result from an upcoming improved API for text. Also note that the "automerge 1.0.1" here is actually the ',(0,o.kt)("inlineCode",{parentName:"p"},"automerge@1.0.1-preview-7")," release. Automerge 1.0.1 was a significant rewrite from 0.14 and has a similar architecture to the Rust implementation. Improvements between 1.0.1 and 2.0.1 are a result of both optimization and adopting WebAssembly rather than an architectural change."),(0,o.kt)("h2",{id:"portability--mobile-devices"},"Portability & Mobile Devices"),(0,o.kt)("p",null,"Because the core logic of Automerge is now built in Rust, we're able to port it more easily to a wide variety of environments and bind it to almost any language. We have users today who directly build on Automerge using the Rust APIs (and the helpful ",(0,o.kt)("a",{parentName:"p",href:"https://github.com/automerge/autosurgeon"},"autosurgeon")," library). We also have a ",(0,o.kt)("a",{parentName:"p",href:"https://github.com/automerge/automerge-rs/tree/main/rust/automerge-c"},"C-bindings API")," designed and contributed by Jason Kankiewicz, and are excited to see the ",(0,o.kt)("a",{parentName:"p",href:"https://github.com/automerge/automerge-go"},(0,o.kt)("inlineCode",{parentName:"a"},"automerge-go"))," implementation underway by Conrad Irwin."),(0,o.kt)("p",null,"In the future, we hope to provide bindings for other languages including Swift, Kotlin, and Python. If you're interested in getting involved in those projects please let us know!"),(0,o.kt)("p",null,"One important note is that React-Native does not support WASM today. Developers building mobile applications will need to bind directly via C. If you're interested in either working on or sponsoring work on this problem, feel free to get in touch."),(0,o.kt)("h1",{id:"whats-next"},"What\u2019s Next"),(0,o.kt)("p",null,"With the release of Automerge 2.0 out the door, we will of course be listening closely to the community about their experience with the release, but in the months ahead, we expect to work on at least some of the following features:"),(0,o.kt)("h2",{id:"native-rich-text-support"},"Native Rich Text Support"),(0,o.kt)("p",null,"As with most CRDTs, Automerge originally focused on optimizing editing of plaintext. In the ",(0,o.kt)("a",{parentName:"p",href:"https://www.inkandswitch.com/peritext/"},"Peritext paper")," by Ink & Switch we discuss an algorithm for supporting rich text with good merging accuracy, and we are planning to integrate this algorithm into Automerge. Support for rich text will also make it easier to implement features like comments or cursor and selection sharing."),(0,o.kt)("h2",{id:"automerge-repo"},"Automerge-Repo"),(0,o.kt)("p",null,"We\u2019ve worked hard to keep Automerge platform-agnostic and support a wide variety of deployment environments. We don\u2019t require a particular network stack or storage system, and Automerge has been used successfully in, client-server web applications, peer-to-peer desktop software, and as a data synchronization engine for cloud services. Unfortunately, excluding network and storage from the library has left a lot of the busy-work up to application developers, and asked them to learn a lot about distributed systems just to get started."),(0,o.kt)("p",null,"Our new library, ",(0,o.kt)("a",{parentName:"p",href:"https://github.com/automerge/automerge-repo"},"Automerge-Repo"),", is a modular batteries-included approach to building web applications with Automerge. It works both in the browser (desktop and mobile) and in Node, and supports a variety of networking and storage adapters. There are even text editor bindings for Quill and Prosemirror as well as React Hooks to make it easy to get started quickly."),(0,o.kt)("p",null,"It's under active development, and available in beta right now. We'll talk more about it when we announce GA, but if you're starting a browser-based application now, it's probably the right place to start."),(0,o.kt)("h2",{id:"rust-developer-experience-improvements"},"Rust Developer Experience Improvements"),(0,o.kt)("p",null,"We've seen tremendous enthusiasm for the native Rust experience of Automerge, and the current Rust API is powerful and fast. Unfortunately, it's also low-level and can be difficult to work with directly. To make building Rust applications against automerge easier, Alex built ",(0,o.kt)("a",{parentName:"p",href:"https://github.com/automerge/autosurgeon"},"Autosurgeon"),", a library that helps bind Rust data structures to Automerge documents, and we'll continue to listen to our Rust users and improve on that experience."),(0,o.kt)("h2",{id:"improved-synchronization"},"Improved Synchronization"),(0,o.kt)("p",null,"Automerge's current synchronization system has some great properties. In many cases it can bring two clients up to date with only a single round-trip each direction. That said, we see big potential to improve the CPU performance of this process, and also lots of opportunity to improve sync performance of many documents at once. We also expect to provide other optimizations our users and sponsors have requested, such as more efficient first-document loading, network compaction of related changes, and enabling something akin to a Git \u201cshallow clone\u201d for clients which don't need historical data."),(0,o.kt)("h2",{id:"built-in-branches"},"Built-in Branches"),(0,o.kt)("p",null,"While we retain the full history of Automerge documents and provide APIs to access it, we don\u2019t currently provide an efficient way to reconcile many closely related versions of a given document. This feature is particularly valuable for supporting offline collaboration in professional environments and (combined with Rich Text Support) should make it much easier for our friends in journalism organizations to build powerful and accurate editing tools."),(0,o.kt)("h2",{id:"history-management"},"History Management"),(0,o.kt)("p",null,"Today the best way to remove something from an Automerge document's history is to recreate the document from scratch or to reset to a time before that change went in. In the future, we plan to provide additional tools to give developers more control over document history. We expect this to include the ability to share just the latest version of a document (similar to a shallow clone in ",(0,o.kt)("inlineCode",{parentName:"p"},"git"),"), and to share updates that bypass changes you don't want to share (as when a developer squashes commits before publishing). "),(0,o.kt)("h1",{id:"conclusion"},"Conclusion"),(0,o.kt)("p",null,"Automerge 2.0 is here, it\u2019s ready for you, and we\u2019re tremendously excited to share it with you. We\u2019ve made Automerge faster, more memory efficient, and we\u2019re bringing it to more platforms than ever. We\u2019re adding features, making it easier to adopt, and have begun growing a team to support it. There has never been a better moment to start building local-first software: why not ",(0,o.kt)("a",{parentName:"p",href:"https://automerge.org/docs/hello/"},"give it a try"),", and please feel welcome to ",(0,o.kt)("a",{parentName:"p",href:"https://join.slack.com/t/automerge/shared_invite/zt-e4p3760n-kKh7r3KRH1YwwNfiZM8ktw"},"join us in the Automerge Slack"),", too."),(0,o.kt)("admonition",{type:"caution"},(0,o.kt)("p",{parentName:"admonition"},"A note to existing users: Automerge 2.0 is found on npm at ",(0,o.kt)("inlineCode",{parentName:"p"},"@automerge/automerge"),". We have deprecated the ",(0,o.kt)("inlineCode",{parentName:"p"},"automerge")," package.")))}d.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/1fd5fe2b.080cc2fe.js b/assets/js/1fd5fe2b.080cc2fe.js new file mode 100644 index 00000000..52edc046 --- /dev/null +++ b/assets/js/1fd5fe2b.080cc2fe.js @@ -0,0 +1 @@ +"use strict";(self.webpackChunkdocs=self.webpackChunkdocs||[]).push([[7044],{3905:(e,t,o)=>{o.d(t,{Zo:()=>d,kt:()=>h});var n=o(7294);function a(e,t,o){return t in e?Object.defineProperty(e,t,{value:o,enumerable:!0,configurable:!0,writable:!0}):e[t]=o,e}function r(e,t){var o=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),o.push.apply(o,n)}return o}function i(e){for(var t=1;t=0||(a[o]=e[o]);return a}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,o)&&(a[o]=e[o])}return a}var s=n.createContext({}),p=function(e){var t=n.useContext(s),o=t;return e&&(o="function"==typeof e?e(t):i(i({},t),e)),o},d=function(e){var t=p(e.components);return n.createElement(s.Provider,{value:t},e.children)},c="mdxType",u={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},m=n.forwardRef((function(e,t){var o=e.components,a=e.mdxType,r=e.originalType,s=e.parentName,d=l(e,["components","mdxType","originalType","parentName"]),c=p(o),m=a,h=c["".concat(s,".").concat(m)]||c[m]||u[m]||r;return o?n.createElement(h,i(i({ref:t},d),{},{components:o})):n.createElement(h,i({ref:t},d))}));function h(e,t){var o=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var r=o.length,i=new Array(r);i[0]=m;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l[c]="string"==typeof e?e:a,i[1]=l;for(var p=2;p{o.r(t),o.d(t,{assets:()=>s,contentTitle:()=>i,default:()=>u,frontMatter:()=>r,metadata:()=>l,toc:()=>p});var n=o(7462),a=(o(7294),o(3905));const r={},i='Automerge-Repo: A "batteries-included" toolkit for building local-first applications',l={permalink:"/blog/2023/11/06/automerge-repo",editUrl:"https://github.com/automerge/automerge.github.io/edit/main/blog/2023-11-06-automerge-repo.md",source:"@site/blog/2023-11-06-automerge-repo.md",title:'Automerge-Repo: A "batteries-included" toolkit for building local-first applications',description:"Today we are announcing our new library, automerge-repo, which makes it vastly easier to build local-first applications with Automerge.",date:"2023-11-06T00:00:00.000Z",formattedDate:"November 6, 2023",tags:[],readingTime:9.165,hasTruncateMarker:!1,authors:[],frontMatter:{},nextItem:{title:"Automerge 2.0",permalink:"/blog/automerge-2"}},s={authorsImageUrls:[]},p=[{value:"automerge-repo: a simple example",id:"automerge-repo-a-simple-example",level:2},{value:"Key Concepts & Basic Usage",id:"key-concepts--basic-usage",level:2},{value:"Repo",id:"repo",level:3},{value:"Handle & Automerge URLs",id:"handle--automerge-urls",level:3},{value:"DocHandle.doc() and DocHandle.docSync()",id:"dochandledoc-and-dochandledocsync",level:3},{value:"change() and on("change")",id:"change-and-onchange",level:3},{value:"Integrations",id:"integrations",level:2},{value:"React Integration",id:"react-integration",level:3},{value:"Svelte Integration",id:"svelte-integration",level:3},{value:"What about <X>?",id:"what-about-x",level:2},{value:"Extending automerge-repo",id:"extending-automerge-repo",level:2},{value:"Storage Adapters",id:"storage-adapters",level:3},{value:"Network Adapters",id:"network-adapters",level:3},{value:"Other languages/platforms",id:"other-languagesplatforms",level:3},{value:"Beta Quality",id:"beta-quality",level:2}],d={toc:p},c="wrapper";function u(e){let{components:t,...o}=e;return(0,a.kt)(c,(0,n.Z)({},d,o,{components:t,mdxType:"MDXLayout"}),(0,a.kt)("p",null,"Today we are announcing our new library, ",(0,a.kt)("a",{parentName:"p",href:"https://github.com/automerge/automerge-repo"},(0,a.kt)("inlineCode",{parentName:"a"},"automerge-repo")),", which makes it vastly easier to build local-first applications with Automerge."),(0,a.kt)("p",null,"For those new to this idea: local-first applications are a way of building software that allows both real-time collaboration (think Google Docs) and offline working (think Git). They work by storing the user's data locally, on their own device, and syncing it with collaborators in the background. You can read more about the motivation for local-first software ",(0,a.kt)("a",{parentName:"p",href:"https://inkandswitch.com/local-first/"},"in our essay"),", or watch a ",(0,a.kt)("a",{parentName:"p",href:"https://www.youtube.com/watch?v=PHz17gwiOc8"},"talk introducing the idea"),"."),(0,a.kt)("p",null,"A challenge in local-first software is how to merge edits that were made independently on different devices, and ",(0,a.kt)("a",{parentName:"p",href:"https://crdt.tech/"},"CRDTs")," were developed to solve this problem. Automerge is a fairly mature CRDT implementation. In fact, we wrote this blog post using it! The API is quite low-level though, and Automerge-Core has no opinion about how networking or storage should be done. Often, the first thing developers ask after discovering Automerge was how to connect it into an actual application."),(0,a.kt)("p",null,"Our new library, ",(0,a.kt)("inlineCode",{parentName:"p"},"automerge-repo"),", extends the collaboration engine of Automerge-Core with networking and storage adapters, and provides integrations with React and other UI frameworks. You can get to building your app straight away by taking advantage of default implementations that solve common problems such as how to send binary data over a WebSocket, how often to send synchronization messages, what network format to use, or how to store data in places like the browser's IndexedDB or on the filesystem."),(0,a.kt)("p",null,"If you've been intimidated by the effort of integrating Automerge into your application because of these choices, this library is for you. Now you can simply create a repo, point it to a sync server, and get to work on your app."),(0,a.kt)("h2",{id:"automerge-repo-a-simple-example"},(0,a.kt)("inlineCode",{parentName:"h2"},"automerge-repo"),": a simple example"),(0,a.kt)("p",null,"Let's start by taking a look at a simple example of how ",(0,a.kt)("inlineCode",{parentName:"p"},"automerge-repo")," works. To begin, create and configure a repository for Automerge documents."),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre"},'const repo = new Repo({\n storage: new IndexedDBStorageAdapter("automerge-demo"),\n network: [new WebsocketClientNetworkAdapter("wss://sync.automerge.org")]\n})\n')),(0,a.kt)("p",null,"The code in the example above creates a repository and adds a storage and network adapter to it. It tells ",(0,a.kt)("inlineCode",{parentName:"p"},"automerge-repo")," to store all changes in an IndexedDB table called ",(0,a.kt)("inlineCode",{parentName:"p"},"automerge-demo")," and to synchronize documents with the WebSocket server at ",(0,a.kt)("inlineCode",{parentName:"p"},"sync.automerge.org"),". The library is designed to support a wide variety of network transports, and we include a simple client/server WebSocket adapter out of the box. Members of the community are already adding support for other transports, such as WebRTC."),(0,a.kt)("p",null,"In this example we're connecting to the public test server hosted by the Automerge team, but you can also run your own sync server. In fact, our ",(0,a.kt)("a",{parentName:"p",href:"https://github.com/automerge/automerge-repo-sync-server"},"sync server")," runs almost the same code as above, but with a different network and storage adapter."),(0,a.kt)("admonition",{type:"note"},(0,a.kt)("p",{parentName:"admonition"},"The Automerge project provides a public sync server for you to experiment with ",(0,a.kt)("inlineCode",{parentName:"p"},"sync.automerge.org"),". This is not a private instance, and as an experimental service has no reliability or data safety guarantees. Basically, it's good for demos and prototyping, but run your own sync server for production uses.")),(0,a.kt)("p",null,"Next, create a document and make some changes to it:"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre"},' > const handle = repo.create()\n > handle.change(doc => { doc.hello = "World." })\n > console.log(handle.url)\n automerge:2j9knpCseyhnK8izDmLpGP5WMdZQ\n')),(0,a.kt)("p",null,"The code logs a URL to the document you created. On another computer, or in another browser, you could load this document using the same URL, as shown below:"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre"},' > const handle = repo.find("automerge:2j9knpCseyhnK8izDmLpGP5WMdZQ")\n > console.log(await handle.doc())\n // why don\'t you try it and find out?\n')),(0,a.kt)("p",null,"What's happening here to make all this work? ",(0,a.kt)("inlineCode",{parentName:"p"},"automerge-repo")," wraps the core Automerge library and handles all the work of moving the bytes around to make your application function."),(0,a.kt)("h2",{id:"key-concepts--basic-usage"},"Key Concepts & Basic Usage"),(0,a.kt)("p",null,"Let's go into a bit more detail. For full documentation please see ",(0,a.kt)("a",{parentName:"p",href:"https://automerge.org/docs/repositories/"},"the docs"),"."),(0,a.kt)("h3",{id:"repo"},"Repo"),(0,a.kt)("p",null,"Create a repo by initializing it with an optional storage plugin and any number of network adapters. These are the options for initializing a repo:"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre"},"export interface RepoConfig {\n // A unique identifier for this peer, the default is a random id\n peerId?: PeerId\n // Something which knows how to store and retrieve binary blobs\n storage?: StorageAdapter\n // Something which knows how to send and receive sync messages\n network: NetworkAdapter[]\n // A function which determines whether to share a document with a peer\n sharePolicy?: SharePolicy\n}\n")),(0,a.kt)("p",null,'Don\'t let the usage of "peer" confuse you into thinking this is limited to peer to peer connectivity, ',(0,a.kt)("inlineCode",{parentName:"p"},"automerge-repo")," works with both client-server and peer-to-peer network transports."),(0,a.kt)("p",null,"The main methods on Repo are ",(0,a.kt)("inlineCode",{parentName:"p"},"find(url)")," and ",(0,a.kt)("inlineCode",{parentName:"p"},"create()"),", both of which return a ",(0,a.kt)("inlineCode",{parentName:"p"},"DocHandle")," you can work with."),(0,a.kt)("h3",{id:"handle--automerge-urls"},"Handle & Automerge URLs"),(0,a.kt)("p",null,"A ",(0,a.kt)("inlineCode",{parentName:"p"},"DocHandle")," is a reference to an Automerge document that a ",(0,a.kt)("inlineCode",{parentName:"p"},"Repo")," syncs and stores . The ",(0,a.kt)("inlineCode",{parentName:"p"},"Repo")," instance saves any changes you make to the document and syncs with connected peers. Likewise, you can listen over the network for to a ",(0,a.kt)("inlineCode",{parentName:"p"},"Repo")," for any changes it received."),(0,a.kt)("p",null,"Each ",(0,a.kt)("inlineCode",{parentName:"p"},"DocHandle")," has a ",(0,a.kt)("inlineCode",{parentName:"p"},".url")," property. This is a string which uniquely identifies a document in the form ",(0,a.kt)("inlineCode",{parentName:"p"},"automerge:"),". Once you have a URL you can use it to request the document from other peers."),(0,a.kt)("h3",{id:"dochandledoc-and-dochandledocsync"},(0,a.kt)("inlineCode",{parentName:"h3"},"DocHandle.doc()")," and ",(0,a.kt)("inlineCode",{parentName:"h3"},"DocHandle.docSync()")),(0,a.kt)("p",null,"These two methods return the current state of the document. ",(0,a.kt)("inlineCode",{parentName:"p"},"doc()")," is an asynchronous method that resolves when a repository loads the document from storage or retrieves it from a peer (whichever happens first), and ",(0,a.kt)("inlineCode",{parentName:"p"},"docSync()")," is a synchronous method that assumes the document is already available.\nThe examples below illustrate asynchronously loading a document or synchronously loading a document and then interacting with it:"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre"},'> const handle = repo.find("automerge:2j9knpCseyhnK8izDmLpGP5WMdZQ")\n> const doc = await handle.doc()\n> console.log(doc)\n')),(0,a.kt)("p",null,"Or "),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre"},'> const handle = repo.find("automerge:2j9knpCseyhnK8izDmLpGP5WMdZQ")\n> handle.whenReady().then(() => {\n console.log(handle.docSync())\n})\n')),(0,a.kt)("p",null,"In this latter example we use ",(0,a.kt)("inlineCode",{parentName:"p"},"DocHandle.whenReady"),", which returns a promise that the repository resolves when it loads a document from storage or fetches it from another peer in the network."),(0,a.kt)("h3",{id:"change-and-onchange"},(0,a.kt)("inlineCode",{parentName:"h3"},"change()")," and ",(0,a.kt)("inlineCode",{parentName:"h3"},'on("change")')),(0,a.kt)("p",null,"Use ",(0,a.kt)("inlineCode",{parentName:"p"},"DocHandle.change")," when you modify a document."),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre"},'> const handle = repo.find("automerge:2j9knpCseyhnK8izDmLpGP5WMdZQ")\n> await handle.doc()\n> handle.change(d => d.foo = "bar")\n')),(0,a.kt)("p",null,"The ",(0,a.kt)("inlineCode",{parentName:"p"},"Repo")," calls ",(0,a.kt)("inlineCode",{parentName:"p"},'DocHandle.on("change")')," whenever the document is modified \u2013 either due to a local change or a sync message being received from another peer."),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre"},'> const handle = repo.find("automerge:4CkUej7mAYnaFMfVnffDipc4Mtvn")\n> await handle.doc()\n> handle.on("change", ({doc}) => {\n console.log("document changed")\n console.log("New content: ", doc)\n})\n')),(0,a.kt)("h2",{id:"integrations"},"Integrations"),(0,a.kt)("p",null,(0,a.kt)("inlineCode",{parentName:"p"},"automerge-repo")," provides a set of primitives that you can use to build a wide range of applications. To make this easier, we have built integrations with a few common UI frameworks. You can easily add further integrations and we welcome contributions which integrate with popular frameworks!"),(0,a.kt)("h3",{id:"react-integration"},"React Integration"),(0,a.kt)("p",null,(0,a.kt)("a",{parentName:"p",href:"https://www.npmjs.com/package/@automerge/automerge-repo-react-hooks"},(0,a.kt)("inlineCode",{parentName:"a"},"@automerge/automerge-repo-react-hooks"))," makes it easy to use ",(0,a.kt)("inlineCode",{parentName:"p"},"automerge-repo")," in a React application. Once you've constructed a ",(0,a.kt)("inlineCode",{parentName:"p"},"Repo")," you can make it available to your React application using ",(0,a.kt)("a",{parentName:"p",href:"https://automerge.org/automerge-repo/variables/_automerge_automerge_repo_react_hooks.RepoContext.html"},(0,a.kt)("inlineCode",{parentName:"a"},"RepoContext")),". Once available, call ",(0,a.kt)("inlineCode",{parentName:"p"},"useHandle")," to obtain a ",(0,a.kt)("inlineCode",{parentName:"p"},"DocHandle"),":"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre"},"function TodoList(listUrl: AutomergeUrl) {\n const handle = useHandle(listUrl)\n // render the todolist\n}\n")),(0,a.kt)("p",null,"Note that when ",(0,a.kt)("inlineCode",{parentName:"p"},"Repo")," receives changes over the network or registers local changes, the original Automerge document remains immutable, and any modified parts of the document get new objects. This means that React will only re-render the parts of the UI that depend on a part of the document that has changed."),(0,a.kt)("h3",{id:"svelte-integration"},"Svelte Integration"),(0,a.kt)("p",null,(0,a.kt)("a",{parentName:"p",href:"https://www.npmjs.com/package/@automerge/automerge-repo-svelte-store"},(0,a.kt)("inlineCode",{parentName:"a"},"@automerge/automerge-repo-svelte-store"))," provides ",(0,a.kt)("inlineCode",{parentName:"p"},"setContextRepo")," to set the ",(0,a.kt)("inlineCode",{parentName:"p"},"Repo")," which is used by the ",(0,a.kt)("inlineCode",{parentName:"p"},"document")," store:"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre"},'
+

Automerge-Repo: A "batteries-included" toolkit for building local-first applications

· 10 min read

Today we are announcing our new library, automerge-repo, which makes it vastly easier to build local-first applications with Automerge.

For those new to this idea: local-first applications are a way of building software that allows both real-time collaboration (think Google Docs) and offline working (think Git). They work by storing the user's data locally, on their own device, and syncing it with collaborators in the background. You can read more about the motivation for local-first software in our essay, or watch a talk introducing the idea.

A challenge in local-first software is how to merge edits that were made independently on different devices, and CRDTs were developed to solve this problem. Automerge is a fairly mature CRDT implementation. In fact, we wrote this blog post using it! The API is quite low-level though, and Automerge-Core has no opinion about how networking or storage should be done. Often, the first thing developers ask after discovering Automerge was how to connect it into an actual application.

Our new library, automerge-repo, extends the collaboration engine of Automerge-Core with networking and storage adapters, and provides integrations with React and other UI frameworks. You can get to building your app straight away by taking advantage of default implementations that solve common problems such as how to send binary data over a WebSocket, how often to send synchronization messages, what network format to use, or how to store data in places like the browser's IndexedDB or on the filesystem.

If you've been intimidated by the effort of integrating Automerge into your application because of these choices, this library is for you. Now you can simply create a repo, point it to a sync server, and get to work on your app.

automerge-repo: a simple example​

Let's start by taking a look at a simple example of how automerge-repo works. To begin, create and configure a repository for Automerge documents.

const repo = new Repo({
  storage: new IndexedDBStorageAdapter("automerge-demo"),
  network: [new WebsocketClientNetworkAdapter("wss://sync.automerge.org")]
})

The code in the example above creates a repository and adds a storage and network adapter to it. It tells automerge-repo to store all changes in an IndexedDB table called automerge-demo and to synchronize documents with the WebSocket server at sync.automerge.org. The library is designed to support a wide variety of network transports, and we include a simple client/server WebSocket adapter out of the box. Members of the community are already adding support for other transports, such as WebRTC.

In this example we're connecting to the public test server hosted by the Automerge team, but you can also run your own sync server. In fact, our sync server runs almost the same code as above, but with a different network and storage adapter.

note

The Automerge project provides a public sync server for you to experiment with sync.automerge.org. This is not a private instance, and as an experimental service has no reliability or data safety guarantees. Basically, it's good for demos and prototyping, but run your own sync server for production uses.

Next, create a document and make some changes to it:

   > const handle = repo.create()
   > handle.change(doc => { doc.hello = "World." })
   > console.log(handle.url)
   automerge:2j9knpCseyhnK8izDmLpGP5WMdZQ

The code logs a URL to the document you created. On another computer, or in another browser, you could load this document using the same URL, as shown below:

   > const handle = repo.find("automerge:2j9knpCseyhnK8izDmLpGP5WMdZQ")
   > console.log(await handle.doc())
   // why don't you try it and find out?

What's happening here to make all this work? automerge-repo wraps the core Automerge library and handles all the work of moving the bytes around to make your application function.

Key Concepts & Basic Usage​

Let's go into a bit more detail. For full documentation please see the docs.

Repo​

Create a repo by initializing it with an optional storage plugin and any number of network adapters. These are the options for initializing a repo:

export interface RepoConfig {
  // A unique identifier for this peer, the default is a random id
  peerId?: PeerId
  // Something which knows how to store and retrieve binary blobs
  storage?: StorageAdapter
  // Something which knows how to send and receive sync messages
  network: NetworkAdapter[]
  // A function which determines whether to share a document with a peer
  sharePolicy?: SharePolicy
}

Don't let the usage of "peer" confuse you into thinking this is limited to peer to peer connectivity, automerge-repo works with both client-server and peer-to-peer network transports.

The main methods on Repo are find(url) and create(), both of which return a DocHandle you can work with.

Handle & Automerge URLs​

A DocHandle is a reference to an Automerge document that a Repo syncs and stores . The Repo instance saves any changes you make to the document and syncs with connected peers. Likewise, you can listen over the network for to a Repo for any changes it received.

Each DocHandle has a .url property. This is a string which uniquely identifies a document in the form automerge:<base58 encoded bytes>. Once you have a URL you can use it to request the document from other peers.

DocHandle.doc() and DocHandle.docSync()​

These two methods return the current state of the document. doc() is an asynchronous method that resolves when a repository loads the document from storage or retrieves it from a peer (whichever happens first), and docSync() is a synchronous method that assumes the document is already available. +The examples below illustrate asynchronously loading a document or synchronously loading a document and then interacting with it:

> const handle = repo.find("automerge:2j9knpCseyhnK8izDmLpGP5WMdZQ")
> const doc = await handle.doc()
> console.log(doc)

Or 

> const handle = repo.find("automerge:2j9knpCseyhnK8izDmLpGP5WMdZQ")
> handle.whenReady().then(() => {
  console.log(handle.docSync())
})

In this latter example we use DocHandle.whenReady, which returns a promise that the repository resolves when it loads a document from storage or fetches it from another peer in the network.

change() and on("change")​

Use DocHandle.change when you modify a document.

> const handle = repo.find("automerge:2j9knpCseyhnK8izDmLpGP5WMdZQ")
> await handle.doc()
> handle.change(d => d.foo = "bar")

The Repo calls DocHandle.on("change") whenever the document is modified – either due to a local change or a sync message being received from another peer.

> const handle = repo.find("automerge:4CkUej7mAYnaFMfVnffDipc4Mtvn")
> await handle.doc()
> handle.on("change", ({doc}) => {
  console.log("document changed")
  console.log("New content: ", doc)
})

Integrations​

automerge-repo provides a set of primitives that you can use to build a wide range of applications. To make this easier, we have built integrations with a few common UI frameworks. You can easily add further integrations and we welcome contributions which integrate with popular frameworks!

React Integration​

@automerge/automerge-repo-react-hooks makes it easy to use automerge-repo in a React application. Once you've constructed a Repo you can make it available to your React application using RepoContext. Once available, call useHandle to obtain a DocHandle:

function TodoList(listUrl: AutomergeUrl) {
    const handle = useHandle(listUrl)
    // render the todolist
}

Note that when Repo receives changes over the network or registers local changes, the original Automerge document remains immutable, and any modified parts of the document get new objects. This means that React will only re-render the parts of the UI that depend on a part of the document that has changed.

Svelte Integration​

@automerge/automerge-repo-svelte-store provides setContextRepo to set the Repo which is used by the document store:

<script lang="ts">
  import { document } from "@automerge/automerge-repo-svelte-store"
  import { type AutomergeUrl } from "@automerge/automerge-repo"

  export let documentUrl: AutomergeUrl

  // Doc is an automerge store with a `change` method which accepts
  // a standard automerge change function
  const doc = document<HasCount>(documentUrl)
  const increment = () => {
    doc.change((d: HasCount) => (d.count = (d.count || 0) + 1))
  }
</script>

<button on:click={increment}>
  count is {$doc?.count || 0}
</button>

What about <X>?​

We'd love to help you make automerge work in your favorite development environment! Please reach out to us on GitHub or via our Slack.

Extending automerge-repo​

You can extend automerge-repo by writing new storage or network adapters.

Storage Adapters​

A storage adapter represents some kind of backend that stores the data in a repo. Storage adapters can be implemented for any key/value store that allows you to query a range of keys with a given prefix. There is no concurrency control required (that's implemented in automerge-repo) so you can safely have multiple repos pointing at the same storage. For example, you could implement an adapter on top of Redis.

The automerge-repo library provides storage adapters for IndexedDB and the file system (on Node).

Network Adapters​

A network adapter represents a way of connecting to other peers. Network adapters raise events when a new peer is discovered or when a message is recieved, and implement a send method for transmitting messages to another peer. automerge-repo assumes a reliable, in-order transport for each peer; as long as you can provide this (e.g. using a TCP connection), you can implement an adapter. You could implement an adapter for BLE, for example.

The automerge-repo library provides network adapters for WebSocket, MessageChannel, and BroadcastChannel.

Other languages/platforms​

This release of automerge-repo is just for javascript. Automerge is a multi-language library though and there are efforts under way to implement automerge-repo on other platforms. The most mature of these is automerge-repo-rs. We welcome contributions and please reach out if you're starting to develop automerge-repo for a new platform.

Beta Quality​

automerge-repo works pretty well – we're using it at Ink & Switch for a bunch of internal projects. The basic shape of the API is simple and useful, and not having to think about the plumbing makes it much, much faster to get a useful application off the ground. However, there are some performance problems we're working on:

  1. Documents with large histories (e.g. a collaboratively edited document with >60,000 edits) can be slow to sync.
  2. The sync protocol currently requires that a document it is syncing be loaded into memory. This means that a sync server can struggle to handle a lot of traffic on large documents.

These two points mean that we're not ready to say this project is ready for production.

We're working hard on fixing the performance so that we can say this is ready for production. But if you are interested in experimenting with the library now, or if you are only going to be working with relatively small documents or low traffic sync servers then you are good to go!

(If you want us to get to production faster, or you have some specific requirements, please consider sponsoring Automerge development 🙂)

Finally, we don't want to give the impression that everything is smooth sailing. automerge-repo solves a bunch of the hard problems people were encountering around networking and storage. There are still plenty of other difficult problems in local first software where we don't have turnkey solutions: authentication and authorization, end-to-end encryption, schema changes, version control workflows etc. automerge-repo makes many things much easier, but it's a frontier out here.

+ + + + \ No newline at end of file diff --git a/blog/archive/index.html b/blog/archive/index.html index 46a674c2..8b3654bc 100644 --- a/blog/archive/index.html +++ b/blog/archive/index.html @@ -10,13 +10,13 @@ - - + +
-

Archive

Archive

2022

2023

- - +

Archive

Archive

2022

2023

+ + \ No newline at end of file diff --git a/blog/atom.xml b/blog/atom.xml index 16936139..82649c70 100644 --- a/blog/atom.xml +++ b/blog/atom.xml @@ -2,11 +2,20 @@ https://automerge.github.io/blog Automerge CRDT Blog - 2023-01-17T00:00:00.000Z + 2023-11-06T00:00:00.000Z https://github.com/jpmonette/feed Automerge CRDT Blog https://automerge.github.io/img/favicon.ico + + <![CDATA[Automerge-Repo: A "batteries-included" toolkit for building local-first applications]]> + https://automerge.github.io/blog/2023/11/06/automerge-repo + + 2023-11-06T00:00:00.000Z + + Today we are announcing our new library, automerge-repo, which makes it vastly easier to build local-first applications with Automerge.

For those new to this idea: local-first applications are a way of building software that allows both real-time collaboration (think Google Docs) and offline working (think Git). They work by storing the user's data locally, on their own device, and syncing it with collaborators in the background. You can read more about the motivation for local-first software in our essay, or watch a talk introducing the idea.

A challenge in local-first software is how to merge edits that were made independently on different devices, and CRDTs were developed to solve this problem. Automerge is a fairly mature CRDT implementation. In fact, we wrote this blog post using it! The API is quite low-level though, and Automerge-Core has no opinion about how networking or storage should be done. Often, the first thing developers ask after discovering Automerge was how to connect it into an actual application.

Our new library, automerge-repo, extends the collaboration engine of Automerge-Core with networking and storage adapters, and provides integrations with React and other UI frameworks. You can get to building your app straight away by taking advantage of default implementations that solve common problems such as how to send binary data over a WebSocket, how often to send synchronization messages, what network format to use, or how to store data in places like the browser's IndexedDB or on the filesystem.

If you've been intimidated by the effort of integrating Automerge into your application because of these choices, this library is for you. Now you can simply create a repo, point it to a sync server, and get to work on your app.

automerge-repo: a simple example​

Let's start by taking a look at a simple example of how automerge-repo works. To begin, create and configure a repository for Automerge documents.

const repo = new Repo({
  storage: new IndexedDBStorageAdapter("automerge-demo"),
  network: [new WebsocketClientNetworkAdapter("wss://sync.automerge.org")]
})

The code in the example above creates a repository and adds a storage and network adapter to it. It tells automerge-repo to store all changes in an IndexedDB table called automerge-demo and to synchronize documents with the WebSocket server at sync.automerge.org. The library is designed to support a wide variety of network transports, and we include a simple client/server WebSocket adapter out of the box. Members of the community are already adding support for other transports, such as WebRTC.

In this example we're connecting to the public test server hosted by the Automerge team, but you can also run your own sync server. In fact, our sync server runs almost the same code as above, but with a different network and storage adapter.

note

The Automerge project provides a public sync server for you to experiment with sync.automerge.org. This is not a private instance, and as an experimental service has no reliability or data safety guarantees. Basically, it's good for demos and prototyping, but run your own sync server for production uses.

Next, create a document and make some changes to it:

   > const handle = repo.create()
   > handle.change(doc => { doc.hello = "World." })
   > console.log(handle.url)
   automerge:2j9knpCseyhnK8izDmLpGP5WMdZQ

The code logs a URL to the document you created. On another computer, or in another browser, you could load this document using the same URL, as shown below:

   > const handle = repo.find("automerge:2j9knpCseyhnK8izDmLpGP5WMdZQ")
   > console.log(await handle.doc())
   // why don't you try it and find out?

What's happening here to make all this work? automerge-repo wraps the core Automerge library and handles all the work of moving the bytes around to make your application function.

Key Concepts & Basic Usage​

Let's go into a bit more detail. For full documentation please see the docs.

Repo​

Create a repo by initializing it with an optional storage plugin and any number of network adapters. These are the options for initializing a repo:

export interface RepoConfig {
  // A unique identifier for this peer, the default is a random id
  peerId?: PeerId
  // Something which knows how to store and retrieve binary blobs
  storage?: StorageAdapter
  // Something which knows how to send and receive sync messages
  network: NetworkAdapter[]
  // A function which determines whether to share a document with a peer
  sharePolicy?: SharePolicy
}

Don't let the usage of "peer" confuse you into thinking this is limited to peer to peer connectivity, automerge-repo works with both client-server and peer-to-peer network transports.

The main methods on Repo are find(url) and create(), both of which return a DocHandle you can work with.

Handle & Automerge URLs​

A DocHandle is a reference to an Automerge document that a Repo syncs and stores . The Repo instance saves any changes you make to the document and syncs with connected peers. Likewise, you can listen over the network for to a Repo for any changes it received.

Each DocHandle has a .url property. This is a string which uniquely identifies a document in the form automerge:<base58 encoded bytes>. Once you have a URL you can use it to request the document from other peers.

DocHandle.doc() and DocHandle.docSync()​

These two methods return the current state of the document. doc() is an asynchronous method that resolves when a repository loads the document from storage or retrieves it from a peer (whichever happens first), and docSync() is a synchronous method that assumes the document is already available. +The examples below illustrate asynchronously loading a document or synchronously loading a document and then interacting with it:

> const handle = repo.find("automerge:2j9knpCseyhnK8izDmLpGP5WMdZQ")
> const doc = await handle.doc()
> console.log(doc)

Or 

> const handle = repo.find("automerge:2j9knpCseyhnK8izDmLpGP5WMdZQ")
> handle.whenReady().then(() => {
  console.log(handle.docSync())
})

In this latter example we use DocHandle.whenReady, which returns a promise that the repository resolves when it loads a document from storage or fetches it from another peer in the network.

change() and on("change")​

Use DocHandle.change when you modify a document.

> const handle = repo.find("automerge:2j9knpCseyhnK8izDmLpGP5WMdZQ")
> await handle.doc()
> handle.change(d => d.foo = "bar")

The Repo calls DocHandle.on("change") whenever the document is modified – either due to a local change or a sync message being received from another peer.

> const handle = repo.find("automerge:4CkUej7mAYnaFMfVnffDipc4Mtvn")
> await handle.doc()
> handle.on("change", ({doc}) => {
  console.log("document changed")
  console.log("New content: ", doc)
})

Integrations​

automerge-repo provides a set of primitives that you can use to build a wide range of applications. To make this easier, we have built integrations with a few common UI frameworks. You can easily add further integrations and we welcome contributions which integrate with popular frameworks!

React Integration​

@automerge/automerge-repo-react-hooks makes it easy to use automerge-repo in a React application. Once you've constructed a Repo you can make it available to your React application using RepoContext. Once available, call useHandle to obtain a DocHandle:

function TodoList(listUrl: AutomergeUrl) {
    const handle = useHandle(listUrl)
    // render the todolist
}

Note that when Repo receives changes over the network or registers local changes, the original Automerge document remains immutable, and any modified parts of the document get new objects. This means that React will only re-render the parts of the UI that depend on a part of the document that has changed.

Svelte Integration​

@automerge/automerge-repo-svelte-store provides setContextRepo to set the Repo which is used by the document store:

<script lang="ts">
  import { document } from "@automerge/automerge-repo-svelte-store"
  import { type AutomergeUrl } from "@automerge/automerge-repo"

  export let documentUrl: AutomergeUrl

  // Doc is an automerge store with a `change` method which accepts
  // a standard automerge change function
  const doc = document<HasCount>(documentUrl)
  const increment = () => {
    doc.change((d: HasCount) => (d.count = (d.count || 0) + 1))
  }
</script>

<button on:click={increment}>
  count is {$doc?.count || 0}
</button>

What about <X>?​

We'd love to help you make automerge work in your favorite development environment! Please reach out to us on GitHub or via our Slack.

Extending automerge-repo​

You can extend automerge-repo by writing new storage or network adapters.

Storage Adapters​

A storage adapter represents some kind of backend that stores the data in a repo. Storage adapters can be implemented for any key/value store that allows you to query a range of keys with a given prefix. There is no concurrency control required (that's implemented in automerge-repo) so you can safely have multiple repos pointing at the same storage. For example, you could implement an adapter on top of Redis.

The automerge-repo library provides storage adapters for IndexedDB and the file system (on Node).

Network Adapters​

A network adapter represents a way of connecting to other peers. Network adapters raise events when a new peer is discovered or when a message is recieved, and implement a send method for transmitting messages to another peer. automerge-repo assumes a reliable, in-order transport for each peer; as long as you can provide this (e.g. using a TCP connection), you can implement an adapter. You could implement an adapter for BLE, for example.

The automerge-repo library provides network adapters for WebSocket, MessageChannel, and BroadcastChannel.

Other languages/platforms​

This release of automerge-repo is just for javascript. Automerge is a multi-language library though and there are efforts under way to implement automerge-repo on other platforms. The most mature of these is automerge-repo-rs. We welcome contributions and please reach out if you're starting to develop automerge-repo for a new platform.

Beta Quality​

automerge-repo works pretty well – we're using it at Ink & Switch for a bunch of internal projects. The basic shape of the API is simple and useful, and not having to think about the plumbing makes it much, much faster to get a useful application off the ground. However, there are some performance problems we're working on:

  1. Documents with large histories (e.g. a collaboratively edited document with >60,000 edits) can be slow to sync.
  2. The sync protocol currently requires that a document it is syncing be loaded into memory. This means that a sync server can struggle to handle a lot of traffic on large documents.

These two points mean that we're not ready to say this project is ready for production.

We're working hard on fixing the performance so that we can say this is ready for production. But if you are interested in experimenting with the library now, or if you are only going to be working with relatively small documents or low traffic sync servers then you are good to go!

(If you want us to get to production faster, or you have some specific requirements, please consider sponsoring Automerge development 🙂)

Finally, we don't want to give the impression that everything is smooth sailing. automerge-repo solves a bunch of the hard problems people were encountering around networking and storage. There are still plenty of other difficult problems in local first software where we don't have turnkey solutions: authentication and authorization, end-to-end encryption, schema changes, version control workflows etc. automerge-repo makes many things much easier, but it's a frontier out here.

]]> + <![CDATA[Automerge 2.0]]> https://automerge.github.io/blog/automerge-2 diff --git a/blog/automerge-2/index.html b/blog/automerge-2/index.html index c154c594..2906347a 100644 --- a/blog/automerge-2/index.html +++ b/blog/automerge-2/index.html @@ -10,14 +10,14 @@ - - + +
-

Automerge 2.0

· 12 min read

Automerge 2.0 is here and ready for production. It’s our first supported release resulting from a ground-up rewrite. The result is a production-ready CRDT with huge improvements in performance and reliability. It's available in both JavaScript and Rust, and includes TypeScript types and C bindings for use in other ecosystems. Even better, Automerge 2.0 comes with improved documentation and, for the first time, support options for production users.

Automerge, CRDTs, and Local-first Software​

Before getting into the details of why we're excited about Automerge 2.0, let's take a bit of time to explain what Automerge is for anyone unfamiliar with the project.

Automerge is a CRDT, or "conflict-free replicated data type", but if you're allergic to buzzwords you can just think of it as a version controlled data structure. Automerge lets you record changes made to data and then replay them in other places, reliably producing the same result in each. It supports JSON-like data, including arbitrarily nested maps and arrays, as well as some more advanced data types such as text and numeric counters.

This is useful for quite a few reasons: you can use it to implement real-time collaboration for an application without having to figure out tricky application-specific algorithms on the server. You can also use it to better support offline work. We think it has even more potential than just that.

Since the rise of the cloud, developers have largely had to choose between building cloud software or traditional installed software. Although installed software has some reliability and performance benefits, cloud software has dominated the market. Cloud software makes sharing data between users easy and includes ubiquitous access from any computing device. Unfortunately, the advantages of cloud software come at a high price. Cloud software is fragile and prone to outages, rarely supports offline use, and is expensive to scale to large audiences.

At Ink & Switch, we’ve been researching a model for developing software which we call local-first software, with the goal of combining the best of both worlds: reliable, locally-executed software paired with scalable offline-friendly collaboration infrastructure. We believe that a strong data model based on recording change over time for every application should be a cornerstone of that effort.

Automerge-RS: Rebuilt for Performance & Portability​

Earlier versions of Automerge were implemented in pure JavaScript. Our initial implementations were theoretically sound but much too slow and used too much memory for most production use cases.

Furthermore, JavaScript support on mobile devices and embedded systems is limited. We wanted a fast and efficient version of Automerge that was available everywhere: in the browser, on any mobile device, and even microcontrollers like the ESP32.

Instead of trying to coordinate multiple distinct versions of Automerge, we decided to rewrite Automerge in Rust and use platform-specific wrappers to make it available in each language ecosystem. This way we can be confident that the core CRDT logic is identical across all platforms and that everyone benefits from new features and optimizations together.

For JavaScript applications, this means compiling the Rust to WebAssembly and providing a JavaScript wrapper that maintains the existing Automerge API. Rust applications can obviously use the library directly, and we're making sure that it's as easy as possible to implement support in other languages with well-designed traits and a comprehensive set of C bindings.

To deliver this new version, lab members Alex Good and Orion Henry teamed up with open source collaborators including Andrew Jeffery and Jason Kankiewicz to polish and optimize the Rust implementation and JavaScript wrapper. The result is a codebase that is hundreds of times faster than past releases, radically more memory efficient, better tested, and more reliable.

Documenting Automerge​

With Automerge 2.0 we've made a big investment in improving documentation. In addition to sample code, we now have a tutorial and quick-start guide that support both Vite and create-react-app, as well as internals documentation, file format and sync protocol documentation. This work was led by lab alumnus Rae McKelvey and we hope it helps make getting started with Automerge much easier. Please let us know if there are other topics or areas you'd like to see covered!

Supporting Automerge​

Those who have been following Automerge for a while may have noticed that we describe Automerge 2.0 as our first supported release. That’s because as part of the Automerge 2.0 release we’ve brought Alex Good onto the team full-time to provide support to external users, handle documentation, release management, and—of course—to continue implementing new Automerge features for the community.

This is a big moment for Ink & Switch and the Automerge project: we’re now able to provide support to our users thanks to sponsorship from enterprises like Fly.io, Prisma, and Bowtie as well as so many others who have contributed either directly to Automerge or through supporting Martin Kleppmann on Patreon.

If your business is interested in sponsoring Automerge, you can sponsor us directly, or get in touch with us for more information or other sponsorship methods. Every little bit helps, and the more sponsors we have, the more work we can do while still remaining an independent open source project.

At Bowtie we support Automerge because it's the best way to achieve the resilliency properties that we're delivering to globally distributed private networks. It's clear to me that our sponsorship has furthered our software, and that this crew are among the best distributed-systems thinkers in the business. --- Issac Kelly, CTO, Bowtie.

Performance: Speed, Memory and Disk​

Using a CRDT inherently comes with overhead: we have to track additional information in order to be able to correctly merge work from different sources. The goal of all CRDT authors is to find the right trade-offs between preserving useful history, reducing CPU overhead, and efficiently storing data in memory and on disk.

With the Automerge project, our goal is to retain the full history of any document and allow an author to reconstruct any point in time on demand. As software developers we're accustomed to having this power: it's hard to imagine version control without history.

With Automerge 2.0, we've brought together an efficient binary data format with fast updates, save, and load performance. Without getting too into the details, we accomplish this by packing data efficiently in memory, ensuring that related data is stored close together for quick retrieval.

Let's take a look at some numbers. One of the most challenging benchmarks for CRDTs is realtime text collaboration. That's because a long editing session can result in hundreds of thousands of individual keystrokes to record and synchronize. Martin Kleppmann recorded the keystrokes that went into writing an academic paper and replaying that data has become a popular benchmark for CRDTs.

Insert ~260k operationsTiming (ms)Memory (bytes)
Automerge 0.14~500,000~1,100,000,000
Automerge 1.0.113,052184,721,408
Automerge 2.0.11,81644,523,520
Yjs1,07410,141,696
Automerge 2.0.2-unstable66122,953,984

Of course, even the most productive authors struggle to type an entire paper quite so quickly. Indeed, writing a paper can occur over months or even years, making both storage size on disk and load performance important as well.

Size on Diskbytes
plain text107,121
automerge 2.0129,062
automerge 0.14146,406,415

The binary format works wonders in this example, encoding a full history for the document with only 30% overhead. That's less than one additional byte per character! The naive JSON encoding often used circa automerge 0.14 could exceed 1,300 bytes per character. If you'd like to learn more about the file format, we have a specification document.

Load ~260k operationsTiming (ms)
Automerge 1.0.1590
Automerge 2.0.1593
Automerge 2.0.2-unstable438

Loading the compressed document is fast as well, ensuring the best possible start-up time.

While we are proud of these results, we will continue to invest in improved performance with each release as you can see with the preliminary numbers for the upcoming Automerge 2.0.2 release.

A few notes about methodology before we move on. The particular implementation we used to run the benchmarks can be found here. These numbers were produced on Ryzen 9 7900X. The "timing" column is how long it takes to apply every single edit in the trace, whilst the "memory" common is the peak memory usage during this process.

The improvements found in "2.0.2-unstable" mostly result from an upcoming improved API for text. Also note that the "automerge 1.0.1" here is actually the automerge@1.0.1-preview-7 release. Automerge 1.0.1 was a significant rewrite from 0.14 and has a similar architecture to the Rust implementation. Improvements between 1.0.1 and 2.0.1 are a result of both optimization and adopting WebAssembly rather than an architectural change.

Portability & Mobile Devices​

Because the core logic of Automerge is now built in Rust, we're able to port it more easily to a wide variety of environments and bind it to almost any language. We have users today who directly build on Automerge using the Rust APIs (and the helpful autosurgeon library). We also have a C-bindings API designed and contributed by Jason Kankiewicz, and are excited to see the automerge-go implementation underway by Conrad Irwin.

In the future, we hope to provide bindings for other languages including Swift, Kotlin, and Python. If you're interested in getting involved in those projects please let us know!

One important note is that React-Native does not support WASM today. Developers building mobile applications will need to bind directly via C. If you're interested in either working on or sponsoring work on this problem, feel free to get in touch.

What’s Next

With the release of Automerge 2.0 out the door, we will of course be listening closely to the community about their experience with the release, but in the months ahead, we expect to work on at least some of the following features:

Native Rich Text Support​

As with most CRDTs, Automerge originally focused on optimizing editing of plaintext. In the Peritext paper by Ink & Switch we discuss an algorithm for supporting rich text with good merging accuracy, and we are planning to integrate this algorithm into Automerge. Support for rich text will also make it easier to implement features like comments or cursor and selection sharing.

Automerge-Repo​

We’ve worked hard to keep Automerge platform-agnostic and support a wide variety of deployment environments. We don’t require a particular network stack or storage system, and Automerge has been used successfully in, client-server web applications, peer-to-peer desktop software, and as a data synchronization engine for cloud services. Unfortunately, excluding network and storage from the library has left a lot of the busy-work up to application developers, and asked them to learn a lot about distributed systems just to get started.

Our new library, Automerge-Repo, is a modular batteries-included approach to building web applications with Automerge. It works both in the browser (desktop and mobile) and in Node, and supports a variety of networking and storage adapters. There are even text editor bindings for Quill and Prosemirror as well as React Hooks to make it easy to get started quickly.

It's under active development, and available in beta right now. We'll talk more about it when we announce GA, but if you're starting a browser-based application now, it's probably the right place to start.

Rust Developer Experience Improvements​

We've seen tremendous enthusiasm for the native Rust experience of Automerge, and the current Rust API is powerful and fast. Unfortunately, it's also low-level and can be difficult to work with directly. To make building Rust applications against automerge easier, Alex built Autosurgeon, a library that helps bind Rust data structures to Automerge documents, and we'll continue to listen to our Rust users and improve on that experience.

Improved Synchronization​

Automerge's current synchronization system has some great properties. In many cases it can bring two clients up to date with only a single round-trip each direction. That said, we see big potential to improve the CPU performance of this process, and also lots of opportunity to improve sync performance of many documents at once. We also expect to provide other optimizations our users and sponsors have requested, such as more efficient first-document loading, network compaction of related changes, and enabling something akin to a Git “shallow clone” for clients which don't need historical data.

Built-in Branches​

While we retain the full history of Automerge documents and provide APIs to access it, we don’t currently provide an efficient way to reconcile many closely related versions of a given document. This feature is particularly valuable for supporting offline collaboration in professional environments and (combined with Rich Text Support) should make it much easier for our friends in journalism organizations to build powerful and accurate editing tools.

History Management​

Today the best way to remove something from an Automerge document's history is to recreate the document from scratch or to reset to a time before that change went in. In the future, we plan to provide additional tools to give developers more control over document history. We expect this to include the ability to share just the latest version of a document (similar to a shallow clone in git), and to share updates that bypass changes you don't want to share (as when a developer squashes commits before publishing).

Conclusion

Automerge 2.0 is here, it’s ready for you, and we’re tremendously excited to share it with you. We’ve made Automerge faster, more memory efficient, and we’re bringing it to more platforms than ever. We’re adding features, making it easier to adopt, and have begun growing a team to support it. There has never been a better moment to start building local-first software: why not give it a try, and please feel welcome to join us in the Automerge Slack, too.

caution

A note to existing users: Automerge 2.0 is found on npm at @automerge/automerge. We have deprecated the automerge package.

- - +

Automerge 2.0

· 12 min read

Automerge 2.0 is here and ready for production. It’s our first supported release resulting from a ground-up rewrite. The result is a production-ready CRDT with huge improvements in performance and reliability. It's available in both JavaScript and Rust, and includes TypeScript types and C bindings for use in other ecosystems. Even better, Automerge 2.0 comes with improved documentation and, for the first time, support options for production users.

Automerge, CRDTs, and Local-first Software​

Before getting into the details of why we're excited about Automerge 2.0, let's take a bit of time to explain what Automerge is for anyone unfamiliar with the project.

Automerge is a CRDT, or "conflict-free replicated data type", but if you're allergic to buzzwords you can just think of it as a version controlled data structure. Automerge lets you record changes made to data and then replay them in other places, reliably producing the same result in each. It supports JSON-like data, including arbitrarily nested maps and arrays, as well as some more advanced data types such as text and numeric counters.

This is useful for quite a few reasons: you can use it to implement real-time collaboration for an application without having to figure out tricky application-specific algorithms on the server. You can also use it to better support offline work. We think it has even more potential than just that.

Since the rise of the cloud, developers have largely had to choose between building cloud software or traditional installed software. Although installed software has some reliability and performance benefits, cloud software has dominated the market. Cloud software makes sharing data between users easy and includes ubiquitous access from any computing device. Unfortunately, the advantages of cloud software come at a high price. Cloud software is fragile and prone to outages, rarely supports offline use, and is expensive to scale to large audiences.

At Ink & Switch, we’ve been researching a model for developing software which we call local-first software, with the goal of combining the best of both worlds: reliable, locally-executed software paired with scalable offline-friendly collaboration infrastructure. We believe that a strong data model based on recording change over time for every application should be a cornerstone of that effort.

Automerge-RS: Rebuilt for Performance & Portability​

Earlier versions of Automerge were implemented in pure JavaScript. Our initial implementations were theoretically sound but much too slow and used too much memory for most production use cases.

Furthermore, JavaScript support on mobile devices and embedded systems is limited. We wanted a fast and efficient version of Automerge that was available everywhere: in the browser, on any mobile device, and even microcontrollers like the ESP32.

Instead of trying to coordinate multiple distinct versions of Automerge, we decided to rewrite Automerge in Rust and use platform-specific wrappers to make it available in each language ecosystem. This way we can be confident that the core CRDT logic is identical across all platforms and that everyone benefits from new features and optimizations together.

For JavaScript applications, this means compiling the Rust to WebAssembly and providing a JavaScript wrapper that maintains the existing Automerge API. Rust applications can obviously use the library directly, and we're making sure that it's as easy as possible to implement support in other languages with well-designed traits and a comprehensive set of C bindings.

To deliver this new version, lab members Alex Good and Orion Henry teamed up with open source collaborators including Andrew Jeffery and Jason Kankiewicz to polish and optimize the Rust implementation and JavaScript wrapper. The result is a codebase that is hundreds of times faster than past releases, radically more memory efficient, better tested, and more reliable.

Documenting Automerge​

With Automerge 2.0 we've made a big investment in improving documentation. In addition to sample code, we now have a tutorial and quick-start guide that support both Vite and create-react-app, as well as internals documentation, file format and sync protocol documentation. This work was led by lab alumnus Rae McKelvey and we hope it helps make getting started with Automerge much easier. Please let us know if there are other topics or areas you'd like to see covered!

Supporting Automerge​

Those who have been following Automerge for a while may have noticed that we describe Automerge 2.0 as our first supported release. That’s because as part of the Automerge 2.0 release we’ve brought Alex Good onto the team full-time to provide support to external users, handle documentation, release management, and—of course—to continue implementing new Automerge features for the community.

This is a big moment for Ink & Switch and the Automerge project: we’re now able to provide support to our users thanks to sponsorship from enterprises like Fly.io, Prisma, and Bowtie as well as so many others who have contributed either directly to Automerge or through supporting Martin Kleppmann on Patreon.

If your business is interested in sponsoring Automerge, you can sponsor us directly, or get in touch with us for more information or other sponsorship methods. Every little bit helps, and the more sponsors we have, the more work we can do while still remaining an independent open source project.

At Bowtie we support Automerge because it's the best way to achieve the resilliency properties that we're delivering to globally distributed private networks. It's clear to me that our sponsorship has furthered our software, and that this crew are among the best distributed-systems thinkers in the business. +-- Issac Kelly, CTO, Bowtie.

Performance: Speed, Memory and Disk​

Using a CRDT inherently comes with overhead: we have to track additional information in order to be able to correctly merge work from different sources. The goal of all CRDT authors is to find the right trade-offs between preserving useful history, reducing CPU overhead, and efficiently storing data in memory and on disk.

With the Automerge project, our goal is to retain the full history of any document and allow an author to reconstruct any point in time on demand. As software developers we're accustomed to having this power: it's hard to imagine version control without history.

With Automerge 2.0, we've brought together an efficient binary data format with fast updates, save, and load performance. Without getting too into the details, we accomplish this by packing data efficiently in memory, ensuring that related data is stored close together for quick retrieval.

Let's take a look at some numbers. One of the most challenging benchmarks for CRDTs is realtime text collaboration. That's because a long editing session can result in hundreds of thousands of individual keystrokes to record and synchronize. Martin Kleppmann recorded the keystrokes that went into writing an academic paper and replaying that data has become a popular benchmark for CRDTs.

Insert ~260k operationsTiming (ms)Memory (bytes)
Automerge 0.14~500,000~1,100,000,000
Automerge 1.0.113,052184,721,408
Automerge 2.0.11,81644,523,520
Yjs1,07410,141,696
Automerge 2.0.2-unstable66122,953,984

Of course, even the most productive authors struggle to type an entire paper quite so quickly. Indeed, writing a paper can occur over months or even years, making both storage size on disk and load performance important as well.

Size on Diskbytes
plain text107,121
automerge 2.0129,062
automerge 0.14146,406,415

The binary format works wonders in this example, encoding a full history for the document with only 30% overhead. That's less than one additional byte per character! The naive JSON encoding often used circa automerge 0.14 could exceed 1,300 bytes per character. If you'd like to learn more about the file format, we have a specification document.

Load ~260k operationsTiming (ms)
Automerge 1.0.1590
Automerge 2.0.1593
Automerge 2.0.2-unstable438

Loading the compressed document is fast as well, ensuring the best possible start-up time.

While we are proud of these results, we will continue to invest in improved performance with each release as you can see with the preliminary numbers for the upcoming Automerge 2.0.2 release.

A few notes about methodology before we move on. The particular implementation we used to run the benchmarks can be found here. These numbers were produced on Ryzen 9 7900X. The "timing" column is how long it takes to apply every single edit in the trace, whilst the "memory" common is the peak memory usage during this process.

The improvements found in "2.0.2-unstable" mostly result from an upcoming improved API for text. Also note that the "automerge 1.0.1" here is actually the automerge@1.0.1-preview-7 release. Automerge 1.0.1 was a significant rewrite from 0.14 and has a similar architecture to the Rust implementation. Improvements between 1.0.1 and 2.0.1 are a result of both optimization and adopting WebAssembly rather than an architectural change.

Portability & Mobile Devices​

Because the core logic of Automerge is now built in Rust, we're able to port it more easily to a wide variety of environments and bind it to almost any language. We have users today who directly build on Automerge using the Rust APIs (and the helpful autosurgeon library). We also have a C-bindings API designed and contributed by Jason Kankiewicz, and are excited to see the automerge-go implementation underway by Conrad Irwin.

In the future, we hope to provide bindings for other languages including Swift, Kotlin, and Python. If you're interested in getting involved in those projects please let us know!

One important note is that React-Native does not support WASM today. Developers building mobile applications will need to bind directly via C. If you're interested in either working on or sponsoring work on this problem, feel free to get in touch.

What’s Next

With the release of Automerge 2.0 out the door, we will of course be listening closely to the community about their experience with the release, but in the months ahead, we expect to work on at least some of the following features:

Native Rich Text Support​

As with most CRDTs, Automerge originally focused on optimizing editing of plaintext. In the Peritext paper by Ink & Switch we discuss an algorithm for supporting rich text with good merging accuracy, and we are planning to integrate this algorithm into Automerge. Support for rich text will also make it easier to implement features like comments or cursor and selection sharing.

Automerge-Repo​

We’ve worked hard to keep Automerge platform-agnostic and support a wide variety of deployment environments. We don’t require a particular network stack or storage system, and Automerge has been used successfully in, client-server web applications, peer-to-peer desktop software, and as a data synchronization engine for cloud services. Unfortunately, excluding network and storage from the library has left a lot of the busy-work up to application developers, and asked them to learn a lot about distributed systems just to get started.

Our new library, Automerge-Repo, is a modular batteries-included approach to building web applications with Automerge. It works both in the browser (desktop and mobile) and in Node, and supports a variety of networking and storage adapters. There are even text editor bindings for Quill and Prosemirror as well as React Hooks to make it easy to get started quickly.

It's under active development, and available in beta right now. We'll talk more about it when we announce GA, but if you're starting a browser-based application now, it's probably the right place to start.

Rust Developer Experience Improvements​

We've seen tremendous enthusiasm for the native Rust experience of Automerge, and the current Rust API is powerful and fast. Unfortunately, it's also low-level and can be difficult to work with directly. To make building Rust applications against automerge easier, Alex built Autosurgeon, a library that helps bind Rust data structures to Automerge documents, and we'll continue to listen to our Rust users and improve on that experience.

Improved Synchronization​

Automerge's current synchronization system has some great properties. In many cases it can bring two clients up to date with only a single round-trip each direction. That said, we see big potential to improve the CPU performance of this process, and also lots of opportunity to improve sync performance of many documents at once. We also expect to provide other optimizations our users and sponsors have requested, such as more efficient first-document loading, network compaction of related changes, and enabling something akin to a Git “shallow clone” for clients which don't need historical data.

Built-in Branches​

While we retain the full history of Automerge documents and provide APIs to access it, we don’t currently provide an efficient way to reconcile many closely related versions of a given document. This feature is particularly valuable for supporting offline collaboration in professional environments and (combined with Rich Text Support) should make it much easier for our friends in journalism organizations to build powerful and accurate editing tools.

History Management​

Today the best way to remove something from an Automerge document's history is to recreate the document from scratch or to reset to a time before that change went in. In the future, we plan to provide additional tools to give developers more control over document history. We expect this to include the ability to share just the latest version of a document (similar to a shallow clone in git), and to share updates that bypass changes you don't want to share (as when a developer squashes commits before publishing).

Conclusion

Automerge 2.0 is here, it’s ready for you, and we’re tremendously excited to share it with you. We’ve made Automerge faster, more memory efficient, and we’re bringing it to more platforms than ever. We’re adding features, making it easier to adopt, and have begun growing a team to support it. There has never been a better moment to start building local-first software: why not give it a try, and please feel welcome to join us in the Automerge Slack, too.

caution

A note to existing users: Automerge 2.0 is found on npm at @automerge/automerge. We have deprecated the automerge package.

+ + \ No newline at end of file diff --git a/blog/index.html b/blog/index.html index b9be110c..7887d5b8 100644 --- a/blog/index.html +++ b/blog/index.html @@ -10,14 +10,15 @@ - - + +
-

· 12 min read

Automerge 2.0 is here and ready for production. It’s our first supported release resulting from a ground-up rewrite. The result is a production-ready CRDT with huge improvements in performance and reliability. It's available in both JavaScript and Rust, and includes TypeScript types and C bindings for use in other ecosystems. Even better, Automerge 2.0 comes with improved documentation and, for the first time, support options for production users.

Automerge, CRDTs, and Local-first Software​

Before getting into the details of why we're excited about Automerge 2.0, let's take a bit of time to explain what Automerge is for anyone unfamiliar with the project.

Automerge is a CRDT, or "conflict-free replicated data type", but if you're allergic to buzzwords you can just think of it as a version controlled data structure. Automerge lets you record changes made to data and then replay them in other places, reliably producing the same result in each. It supports JSON-like data, including arbitrarily nested maps and arrays, as well as some more advanced data types such as text and numeric counters.

This is useful for quite a few reasons: you can use it to implement real-time collaboration for an application without having to figure out tricky application-specific algorithms on the server. You can also use it to better support offline work. We think it has even more potential than just that.

Since the rise of the cloud, developers have largely had to choose between building cloud software or traditional installed software. Although installed software has some reliability and performance benefits, cloud software has dominated the market. Cloud software makes sharing data between users easy and includes ubiquitous access from any computing device. Unfortunately, the advantages of cloud software come at a high price. Cloud software is fragile and prone to outages, rarely supports offline use, and is expensive to scale to large audiences.

At Ink & Switch, we’ve been researching a model for developing software which we call local-first software, with the goal of combining the best of both worlds: reliable, locally-executed software paired with scalable offline-friendly collaboration infrastructure. We believe that a strong data model based on recording change over time for every application should be a cornerstone of that effort.

Automerge-RS: Rebuilt for Performance & Portability​

Earlier versions of Automerge were implemented in pure JavaScript. Our initial implementations were theoretically sound but much too slow and used too much memory for most production use cases.

Furthermore, JavaScript support on mobile devices and embedded systems is limited. We wanted a fast and efficient version of Automerge that was available everywhere: in the browser, on any mobile device, and even microcontrollers like the ESP32.

Instead of trying to coordinate multiple distinct versions of Automerge, we decided to rewrite Automerge in Rust and use platform-specific wrappers to make it available in each language ecosystem. This way we can be confident that the core CRDT logic is identical across all platforms and that everyone benefits from new features and optimizations together.

For JavaScript applications, this means compiling the Rust to WebAssembly and providing a JavaScript wrapper that maintains the existing Automerge API. Rust applications can obviously use the library directly, and we're making sure that it's as easy as possible to implement support in other languages with well-designed traits and a comprehensive set of C bindings.

To deliver this new version, lab members Alex Good and Orion Henry teamed up with open source collaborators including Andrew Jeffery and Jason Kankiewicz to polish and optimize the Rust implementation and JavaScript wrapper. The result is a codebase that is hundreds of times faster than past releases, radically more memory efficient, better tested, and more reliable.

Documenting Automerge​

With Automerge 2.0 we've made a big investment in improving documentation. In addition to sample code, we now have a tutorial and quick-start guide that support both Vite and create-react-app, as well as internals documentation, file format and sync protocol documentation. This work was led by lab alumnus Rae McKelvey and we hope it helps make getting started with Automerge much easier. Please let us know if there are other topics or areas you'd like to see covered!

Supporting Automerge​

Those who have been following Automerge for a while may have noticed that we describe Automerge 2.0 as our first supported release. That’s because as part of the Automerge 2.0 release we’ve brought Alex Good onto the team full-time to provide support to external users, handle documentation, release management, and—of course—to continue implementing new Automerge features for the community.

This is a big moment for Ink & Switch and the Automerge project: we’re now able to provide support to our users thanks to sponsorship from enterprises like Fly.io, Prisma, and Bowtie as well as so many others who have contributed either directly to Automerge or through supporting Martin Kleppmann on Patreon.

If your business is interested in sponsoring Automerge, you can sponsor us directly, or get in touch with us for more information or other sponsorship methods. Every little bit helps, and the more sponsors we have, the more work we can do while still remaining an independent open source project.

At Bowtie we support Automerge because it's the best way to achieve the resilliency properties that we're delivering to globally distributed private networks. It's clear to me that our sponsorship has furthered our software, and that this crew are among the best distributed-systems thinkers in the business. +

· 10 min read

Today we are announcing our new library, automerge-repo, which makes it vastly easier to build local-first applications with Automerge.

For those new to this idea: local-first applications are a way of building software that allows both real-time collaboration (think Google Docs) and offline working (think Git). They work by storing the user's data locally, on their own device, and syncing it with collaborators in the background. You can read more about the motivation for local-first software in our essay, or watch a talk introducing the idea.

A challenge in local-first software is how to merge edits that were made independently on different devices, and CRDTs were developed to solve this problem. Automerge is a fairly mature CRDT implementation. In fact, we wrote this blog post using it! The API is quite low-level though, and Automerge-Core has no opinion about how networking or storage should be done. Often, the first thing developers ask after discovering Automerge was how to connect it into an actual application.

Our new library, automerge-repo, extends the collaboration engine of Automerge-Core with networking and storage adapters, and provides integrations with React and other UI frameworks. You can get to building your app straight away by taking advantage of default implementations that solve common problems such as how to send binary data over a WebSocket, how often to send synchronization messages, what network format to use, or how to store data in places like the browser's IndexedDB or on the filesystem.

If you've been intimidated by the effort of integrating Automerge into your application because of these choices, this library is for you. Now you can simply create a repo, point it to a sync server, and get to work on your app.

automerge-repo: a simple example​

Let's start by taking a look at a simple example of how automerge-repo works. To begin, create and configure a repository for Automerge documents.

const repo = new Repo({
  storage: new IndexedDBStorageAdapter("automerge-demo"),
  network: [new WebsocketClientNetworkAdapter("wss://sync.automerge.org")]
})

The code in the example above creates a repository and adds a storage and network adapter to it. It tells automerge-repo to store all changes in an IndexedDB table called automerge-demo and to synchronize documents with the WebSocket server at sync.automerge.org. The library is designed to support a wide variety of network transports, and we include a simple client/server WebSocket adapter out of the box. Members of the community are already adding support for other transports, such as WebRTC.

In this example we're connecting to the public test server hosted by the Automerge team, but you can also run your own sync server. In fact, our sync server runs almost the same code as above, but with a different network and storage adapter.

note

The Automerge project provides a public sync server for you to experiment with sync.automerge.org. This is not a private instance, and as an experimental service has no reliability or data safety guarantees. Basically, it's good for demos and prototyping, but run your own sync server for production uses.

Next, create a document and make some changes to it:

   > const handle = repo.create()
   > handle.change(doc => { doc.hello = "World." })
   > console.log(handle.url)
   automerge:2j9knpCseyhnK8izDmLpGP5WMdZQ

The code logs a URL to the document you created. On another computer, or in another browser, you could load this document using the same URL, as shown below:

   > const handle = repo.find("automerge:2j9knpCseyhnK8izDmLpGP5WMdZQ")
   > console.log(await handle.doc())
   // why don't you try it and find out?

What's happening here to make all this work? automerge-repo wraps the core Automerge library and handles all the work of moving the bytes around to make your application function.

Key Concepts & Basic Usage​

Let's go into a bit more detail. For full documentation please see the docs.

Repo​

Create a repo by initializing it with an optional storage plugin and any number of network adapters. These are the options for initializing a repo:

export interface RepoConfig {
  // A unique identifier for this peer, the default is a random id
  peerId?: PeerId
  // Something which knows how to store and retrieve binary blobs
  storage?: StorageAdapter
  // Something which knows how to send and receive sync messages
  network: NetworkAdapter[]
  // A function which determines whether to share a document with a peer
  sharePolicy?: SharePolicy
}

Don't let the usage of "peer" confuse you into thinking this is limited to peer to peer connectivity, automerge-repo works with both client-server and peer-to-peer network transports.

The main methods on Repo are find(url) and create(), both of which return a DocHandle you can work with.

Handle & Automerge URLs​

A DocHandle is a reference to an Automerge document that a Repo syncs and stores . The Repo instance saves any changes you make to the document and syncs with connected peers. Likewise, you can listen over the network for to a Repo for any changes it received.

Each DocHandle has a .url property. This is a string which uniquely identifies a document in the form automerge:<base58 encoded bytes>. Once you have a URL you can use it to request the document from other peers.

DocHandle.doc() and DocHandle.docSync()​

These two methods return the current state of the document. doc() is an asynchronous method that resolves when a repository loads the document from storage or retrieves it from a peer (whichever happens first), and docSync() is a synchronous method that assumes the document is already available. +The examples below illustrate asynchronously loading a document or synchronously loading a document and then interacting with it:

> const handle = repo.find("automerge:2j9knpCseyhnK8izDmLpGP5WMdZQ")
> const doc = await handle.doc()
> console.log(doc)

Or 

> const handle = repo.find("automerge:2j9knpCseyhnK8izDmLpGP5WMdZQ")
> handle.whenReady().then(() => {
  console.log(handle.docSync())
})

In this latter example we use DocHandle.whenReady, which returns a promise that the repository resolves when it loads a document from storage or fetches it from another peer in the network.

change() and on("change")​

Use DocHandle.change when you modify a document.

> const handle = repo.find("automerge:2j9knpCseyhnK8izDmLpGP5WMdZQ")
> await handle.doc()
> handle.change(d => d.foo = "bar")

The Repo calls DocHandle.on("change") whenever the document is modified – either due to a local change or a sync message being received from another peer.

> const handle = repo.find("automerge:4CkUej7mAYnaFMfVnffDipc4Mtvn")
> await handle.doc()
> handle.on("change", ({doc}) => {
  console.log("document changed")
  console.log("New content: ", doc)
})

Integrations​

automerge-repo provides a set of primitives that you can use to build a wide range of applications. To make this easier, we have built integrations with a few common UI frameworks. You can easily add further integrations and we welcome contributions which integrate with popular frameworks!

React Integration​

@automerge/automerge-repo-react-hooks makes it easy to use automerge-repo in a React application. Once you've constructed a Repo you can make it available to your React application using RepoContext. Once available, call useHandle to obtain a DocHandle:

function TodoList(listUrl: AutomergeUrl) {
    const handle = useHandle(listUrl)
    // render the todolist
}

Note that when Repo receives changes over the network or registers local changes, the original Automerge document remains immutable, and any modified parts of the document get new objects. This means that React will only re-render the parts of the UI that depend on a part of the document that has changed.

Svelte Integration​

@automerge/automerge-repo-svelte-store provides setContextRepo to set the Repo which is used by the document store:

<script lang="ts">
  import { document } from "@automerge/automerge-repo-svelte-store"
  import { type AutomergeUrl } from "@automerge/automerge-repo"

  export let documentUrl: AutomergeUrl

  // Doc is an automerge store with a `change` method which accepts
  // a standard automerge change function
  const doc = document<HasCount>(documentUrl)
  const increment = () => {
    doc.change((d: HasCount) => (d.count = (d.count || 0) + 1))
  }
</script>

<button on:click={increment}>
  count is {$doc?.count || 0}
</button>

What about <X>?​

We'd love to help you make automerge work in your favorite development environment! Please reach out to us on GitHub or via our Slack.

Extending automerge-repo​

You can extend automerge-repo by writing new storage or network adapters.

Storage Adapters​

A storage adapter represents some kind of backend that stores the data in a repo. Storage adapters can be implemented for any key/value store that allows you to query a range of keys with a given prefix. There is no concurrency control required (that's implemented in automerge-repo) so you can safely have multiple repos pointing at the same storage. For example, you could implement an adapter on top of Redis.

The automerge-repo library provides storage adapters for IndexedDB and the file system (on Node).

Network Adapters​

A network adapter represents a way of connecting to other peers. Network adapters raise events when a new peer is discovered or when a message is recieved, and implement a send method for transmitting messages to another peer. automerge-repo assumes a reliable, in-order transport for each peer; as long as you can provide this (e.g. using a TCP connection), you can implement an adapter. You could implement an adapter for BLE, for example.

The automerge-repo library provides network adapters for WebSocket, MessageChannel, and BroadcastChannel.

Other languages/platforms​

This release of automerge-repo is just for javascript. Automerge is a multi-language library though and there are efforts under way to implement automerge-repo on other platforms. The most mature of these is automerge-repo-rs. We welcome contributions and please reach out if you're starting to develop automerge-repo for a new platform.

Beta Quality​

automerge-repo works pretty well – we're using it at Ink & Switch for a bunch of internal projects. The basic shape of the API is simple and useful, and not having to think about the plumbing makes it much, much faster to get a useful application off the ground. However, there are some performance problems we're working on:

  1. Documents with large histories (e.g. a collaboratively edited document with >60,000 edits) can be slow to sync.
  2. The sync protocol currently requires that a document it is syncing be loaded into memory. This means that a sync server can struggle to handle a lot of traffic on large documents.

These two points mean that we're not ready to say this project is ready for production.

We're working hard on fixing the performance so that we can say this is ready for production. But if you are interested in experimenting with the library now, or if you are only going to be working with relatively small documents or low traffic sync servers then you are good to go!

(If you want us to get to production faster, or you have some specific requirements, please consider sponsoring Automerge development 🙂)

Finally, we don't want to give the impression that everything is smooth sailing. automerge-repo solves a bunch of the hard problems people were encountering around networking and storage. There are still plenty of other difficult problems in local first software where we don't have turnkey solutions: authentication and authorization, end-to-end encryption, schema changes, version control workflows etc. automerge-repo makes many things much easier, but it's a frontier out here.

· 12 min read

Automerge 2.0 is here and ready for production. It’s our first supported release resulting from a ground-up rewrite. The result is a production-ready CRDT with huge improvements in performance and reliability. It's available in both JavaScript and Rust, and includes TypeScript types and C bindings for use in other ecosystems. Even better, Automerge 2.0 comes with improved documentation and, for the first time, support options for production users.

Automerge, CRDTs, and Local-first Software​

Before getting into the details of why we're excited about Automerge 2.0, let's take a bit of time to explain what Automerge is for anyone unfamiliar with the project.

Automerge is a CRDT, or "conflict-free replicated data type", but if you're allergic to buzzwords you can just think of it as a version controlled data structure. Automerge lets you record changes made to data and then replay them in other places, reliably producing the same result in each. It supports JSON-like data, including arbitrarily nested maps and arrays, as well as some more advanced data types such as text and numeric counters.

This is useful for quite a few reasons: you can use it to implement real-time collaboration for an application without having to figure out tricky application-specific algorithms on the server. You can also use it to better support offline work. We think it has even more potential than just that.

Since the rise of the cloud, developers have largely had to choose between building cloud software or traditional installed software. Although installed software has some reliability and performance benefits, cloud software has dominated the market. Cloud software makes sharing data between users easy and includes ubiquitous access from any computing device. Unfortunately, the advantages of cloud software come at a high price. Cloud software is fragile and prone to outages, rarely supports offline use, and is expensive to scale to large audiences.

At Ink & Switch, we’ve been researching a model for developing software which we call local-first software, with the goal of combining the best of both worlds: reliable, locally-executed software paired with scalable offline-friendly collaboration infrastructure. We believe that a strong data model based on recording change over time for every application should be a cornerstone of that effort.

Automerge-RS: Rebuilt for Performance & Portability​

Earlier versions of Automerge were implemented in pure JavaScript. Our initial implementations were theoretically sound but much too slow and used too much memory for most production use cases.

Furthermore, JavaScript support on mobile devices and embedded systems is limited. We wanted a fast and efficient version of Automerge that was available everywhere: in the browser, on any mobile device, and even microcontrollers like the ESP32.

Instead of trying to coordinate multiple distinct versions of Automerge, we decided to rewrite Automerge in Rust and use platform-specific wrappers to make it available in each language ecosystem. This way we can be confident that the core CRDT logic is identical across all platforms and that everyone benefits from new features and optimizations together.

For JavaScript applications, this means compiling the Rust to WebAssembly and providing a JavaScript wrapper that maintains the existing Automerge API. Rust applications can obviously use the library directly, and we're making sure that it's as easy as possible to implement support in other languages with well-designed traits and a comprehensive set of C bindings.

To deliver this new version, lab members Alex Good and Orion Henry teamed up with open source collaborators including Andrew Jeffery and Jason Kankiewicz to polish and optimize the Rust implementation and JavaScript wrapper. The result is a codebase that is hundreds of times faster than past releases, radically more memory efficient, better tested, and more reliable.

Documenting Automerge​

With Automerge 2.0 we've made a big investment in improving documentation. In addition to sample code, we now have a tutorial and quick-start guide that support both Vite and create-react-app, as well as internals documentation, file format and sync protocol documentation. This work was led by lab alumnus Rae McKelvey and we hope it helps make getting started with Automerge much easier. Please let us know if there are other topics or areas you'd like to see covered!

Supporting Automerge​

Those who have been following Automerge for a while may have noticed that we describe Automerge 2.0 as our first supported release. That’s because as part of the Automerge 2.0 release we’ve brought Alex Good onto the team full-time to provide support to external users, handle documentation, release management, and—of course—to continue implementing new Automerge features for the community.

This is a big moment for Ink & Switch and the Automerge project: we’re now able to provide support to our users thanks to sponsorship from enterprises like Fly.io, Prisma, and Bowtie as well as so many others who have contributed either directly to Automerge or through supporting Martin Kleppmann on Patreon.

If your business is interested in sponsoring Automerge, you can sponsor us directly, or get in touch with us for more information or other sponsorship methods. Every little bit helps, and the more sponsors we have, the more work we can do while still remaining an independent open source project.

At Bowtie we support Automerge because it's the best way to achieve the resilliency properties that we're delivering to globally distributed private networks. It's clear to me that our sponsorship has furthered our software, and that this crew are among the best distributed-systems thinkers in the business. -- Issac Kelly, CTO, Bowtie.

Performance: Speed, Memory and Disk​

Using a CRDT inherently comes with overhead: we have to track additional information in order to be able to correctly merge work from different sources. The goal of all CRDT authors is to find the right trade-offs between preserving useful history, reducing CPU overhead, and efficiently storing data in memory and on disk.

With the Automerge project, our goal is to retain the full history of any document and allow an author to reconstruct any point in time on demand. As software developers we're accustomed to having this power: it's hard to imagine version control without history.

With Automerge 2.0, we've brought together an efficient binary data format with fast updates, save, and load performance. Without getting too into the details, we accomplish this by packing data efficiently in memory, ensuring that related data is stored close together for quick retrieval.

Let's take a look at some numbers. One of the most challenging benchmarks for CRDTs is realtime text collaboration. That's because a long editing session can result in hundreds of thousands of individual keystrokes to record and synchronize. Martin Kleppmann recorded the keystrokes that went into writing an academic paper and replaying that data has become a popular benchmark for CRDTs.

Insert ~260k operationsTiming (ms)Memory (bytes)
Automerge 0.14~500,000~1,100,000,000
Automerge 1.0.113,052184,721,408
Automerge 2.0.11,81644,523,520
Yjs1,07410,141,696
Automerge 2.0.2-unstable66122,953,984

Of course, even the most productive authors struggle to type an entire paper quite so quickly. Indeed, writing a paper can occur over months or even years, making both storage size on disk and load performance important as well.

Size on Diskbytes
plain text107,121
automerge 2.0129,062
automerge 0.14146,406,415

The binary format works wonders in this example, encoding a full history for the document with only 30% overhead. That's less than one additional byte per character! The naive JSON encoding often used circa automerge 0.14 could exceed 1,300 bytes per character. If you'd like to learn more about the file format, we have a specification document.

Load ~260k operationsTiming (ms)
Automerge 1.0.1590
Automerge 2.0.1593
Automerge 2.0.2-unstable438

Loading the compressed document is fast as well, ensuring the best possible start-up time.

While we are proud of these results, we will continue to invest in improved performance with each release as you can see with the preliminary numbers for the upcoming Automerge 2.0.2 release.

A few notes about methodology before we move on. The particular implementation we used to run the benchmarks can be found here. These numbers were produced on Ryzen 9 7900X. The "timing" column is how long it takes to apply every single edit in the trace, whilst the "memory" common is the peak memory usage during this process.

The improvements found in "2.0.2-unstable" mostly result from an upcoming improved API for text. Also note that the "automerge 1.0.1" here is actually the automerge@1.0.1-preview-7 release. Automerge 1.0.1 was a significant rewrite from 0.14 and has a similar architecture to the Rust implementation. Improvements between 1.0.1 and 2.0.1 are a result of both optimization and adopting WebAssembly rather than an architectural change.

Portability & Mobile Devices​

Because the core logic of Automerge is now built in Rust, we're able to port it more easily to a wide variety of environments and bind it to almost any language. We have users today who directly build on Automerge using the Rust APIs (and the helpful autosurgeon library). We also have a C-bindings API designed and contributed by Jason Kankiewicz, and are excited to see the automerge-go implementation underway by Conrad Irwin.

In the future, we hope to provide bindings for other languages including Swift, Kotlin, and Python. If you're interested in getting involved in those projects please let us know!

One important note is that React-Native does not support WASM today. Developers building mobile applications will need to bind directly via C. If you're interested in either working on or sponsoring work on this problem, feel free to get in touch.

What’s Next

With the release of Automerge 2.0 out the door, we will of course be listening closely to the community about their experience with the release, but in the months ahead, we expect to work on at least some of the following features:

Native Rich Text Support​

As with most CRDTs, Automerge originally focused on optimizing editing of plaintext. In the Peritext paper by Ink & Switch we discuss an algorithm for supporting rich text with good merging accuracy, and we are planning to integrate this algorithm into Automerge. Support for rich text will also make it easier to implement features like comments or cursor and selection sharing.

Automerge-Repo​

We’ve worked hard to keep Automerge platform-agnostic and support a wide variety of deployment environments. We don’t require a particular network stack or storage system, and Automerge has been used successfully in, client-server web applications, peer-to-peer desktop software, and as a data synchronization engine for cloud services. Unfortunately, excluding network and storage from the library has left a lot of the busy-work up to application developers, and asked them to learn a lot about distributed systems just to get started.

Our new library, Automerge-Repo, is a modular batteries-included approach to building web applications with Automerge. It works both in the browser (desktop and mobile) and in Node, and supports a variety of networking and storage adapters. There are even text editor bindings for Quill and Prosemirror as well as React Hooks to make it easy to get started quickly.

It's under active development, and available in beta right now. We'll talk more about it when we announce GA, but if you're starting a browser-based application now, it's probably the right place to start.

Rust Developer Experience Improvements​

We've seen tremendous enthusiasm for the native Rust experience of Automerge, and the current Rust API is powerful and fast. Unfortunately, it's also low-level and can be difficult to work with directly. To make building Rust applications against automerge easier, Alex built Autosurgeon, a library that helps bind Rust data structures to Automerge documents, and we'll continue to listen to our Rust users and improve on that experience.

Improved Synchronization​

Automerge's current synchronization system has some great properties. In many cases it can bring two clients up to date with only a single round-trip each direction. That said, we see big potential to improve the CPU performance of this process, and also lots of opportunity to improve sync performance of many documents at once. We also expect to provide other optimizations our users and sponsors have requested, such as more efficient first-document loading, network compaction of related changes, and enabling something akin to a Git “shallow clone” for clients which don't need historical data.

Built-in Branches​

While we retain the full history of Automerge documents and provide APIs to access it, we don’t currently provide an efficient way to reconcile many closely related versions of a given document. This feature is particularly valuable for supporting offline collaboration in professional environments and (combined with Rich Text Support) should make it much easier for our friends in journalism organizations to build powerful and accurate editing tools.

History Management​

Today the best way to remove something from an Automerge document's history is to recreate the document from scratch or to reset to a time before that change went in. In the future, we plan to provide additional tools to give developers more control over document history. We expect this to include the ability to share just the latest version of a document (similar to a shallow clone in git), and to share updates that bypass changes you don't want to share (as when a developer squashes commits before publishing).

Conclusion

Automerge 2.0 is here, it’s ready for you, and we’re tremendously excited to share it with you. We’ve made Automerge faster, more memory efficient, and we’re bringing it to more platforms than ever. We’re adding features, making it easier to adopt, and have begun growing a team to support it. There has never been a better moment to start building local-first software: why not give it a try, and please feel welcome to join us in the Automerge Slack, too.

caution

A note to existing users: Automerge 2.0 is found on npm at @automerge/automerge. We have deprecated the automerge package.

· One min read
Rae McKelvey

You've reached the Automerge docs! We're so happy to have you.

We're using Docusaurus. Please help edit the docs on GitHub.

- - + + \ No newline at end of file diff --git a/blog/rss.xml b/blog/rss.xml index b38f204d..0cc59a7b 100644 --- a/blog/rss.xml +++ b/blog/rss.xml @@ -4,10 +4,19 @@ Automerge CRDT Blog https://automerge.github.io/blog Automerge CRDT Blog - Tue, 17 Jan 2023 00:00:00 GMT + Mon, 06 Nov 2023 00:00:00 GMT https://validator.w3.org/feed/docs/rss2.html https://github.com/jpmonette/feed en + + <![CDATA[Automerge-Repo: A "batteries-included" toolkit for building local-first applications]]> + https://automerge.github.io/blog/2023/11/06/automerge-repo + https://automerge.github.io/blog/2023/11/06/automerge-repo + Mon, 06 Nov 2023 00:00:00 GMT + + Today we are announcing our new library, automerge-repo, which makes it vastly easier to build local-first applications with Automerge.

For those new to this idea: local-first applications are a way of building software that allows both real-time collaboration (think Google Docs) and offline working (think Git). They work by storing the user's data locally, on their own device, and syncing it with collaborators in the background. You can read more about the motivation for local-first software in our essay, or watch a talk introducing the idea.

A challenge in local-first software is how to merge edits that were made independently on different devices, and CRDTs were developed to solve this problem. Automerge is a fairly mature CRDT implementation. In fact, we wrote this blog post using it! The API is quite low-level though, and Automerge-Core has no opinion about how networking or storage should be done. Often, the first thing developers ask after discovering Automerge was how to connect it into an actual application.

Our new library, automerge-repo, extends the collaboration engine of Automerge-Core with networking and storage adapters, and provides integrations with React and other UI frameworks. You can get to building your app straight away by taking advantage of default implementations that solve common problems such as how to send binary data over a WebSocket, how often to send synchronization messages, what network format to use, or how to store data in places like the browser's IndexedDB or on the filesystem.

If you've been intimidated by the effort of integrating Automerge into your application because of these choices, this library is for you. Now you can simply create a repo, point it to a sync server, and get to work on your app.

automerge-repo: a simple example​

Let's start by taking a look at a simple example of how automerge-repo works. To begin, create and configure a repository for Automerge documents.

const repo = new Repo({
  storage: new IndexedDBStorageAdapter("automerge-demo"),
  network: [new WebsocketClientNetworkAdapter("wss://sync.automerge.org")]
})

The code in the example above creates a repository and adds a storage and network adapter to it. It tells automerge-repo to store all changes in an IndexedDB table called automerge-demo and to synchronize documents with the WebSocket server at sync.automerge.org. The library is designed to support a wide variety of network transports, and we include a simple client/server WebSocket adapter out of the box. Members of the community are already adding support for other transports, such as WebRTC.

In this example we're connecting to the public test server hosted by the Automerge team, but you can also run your own sync server. In fact, our sync server runs almost the same code as above, but with a different network and storage adapter.

note

The Automerge project provides a public sync server for you to experiment with sync.automerge.org. This is not a private instance, and as an experimental service has no reliability or data safety guarantees. Basically, it's good for demos and prototyping, but run your own sync server for production uses.

Next, create a document and make some changes to it:

   > const handle = repo.create()
   > handle.change(doc => { doc.hello = "World." })
   > console.log(handle.url)
   automerge:2j9knpCseyhnK8izDmLpGP5WMdZQ

The code logs a URL to the document you created. On another computer, or in another browser, you could load this document using the same URL, as shown below:

   > const handle = repo.find("automerge:2j9knpCseyhnK8izDmLpGP5WMdZQ")
   > console.log(await handle.doc())
   // why don't you try it and find out?

What's happening here to make all this work? automerge-repo wraps the core Automerge library and handles all the work of moving the bytes around to make your application function.

Key Concepts & Basic Usage​

Let's go into a bit more detail. For full documentation please see the docs.

Repo​

Create a repo by initializing it with an optional storage plugin and any number of network adapters. These are the options for initializing a repo:

export interface RepoConfig {
  // A unique identifier for this peer, the default is a random id
  peerId?: PeerId
  // Something which knows how to store and retrieve binary blobs
  storage?: StorageAdapter
  // Something which knows how to send and receive sync messages
  network: NetworkAdapter[]
  // A function which determines whether to share a document with a peer
  sharePolicy?: SharePolicy
}

Don't let the usage of "peer" confuse you into thinking this is limited to peer to peer connectivity, automerge-repo works with both client-server and peer-to-peer network transports.

The main methods on Repo are find(url) and create(), both of which return a DocHandle you can work with.

Handle & Automerge URLs​

A DocHandle is a reference to an Automerge document that a Repo syncs and stores . The Repo instance saves any changes you make to the document and syncs with connected peers. Likewise, you can listen over the network for to a Repo for any changes it received.

Each DocHandle has a .url property. This is a string which uniquely identifies a document in the form automerge:<base58 encoded bytes>. Once you have a URL you can use it to request the document from other peers.

DocHandle.doc() and DocHandle.docSync()​

These two methods return the current state of the document. doc() is an asynchronous method that resolves when a repository loads the document from storage or retrieves it from a peer (whichever happens first), and docSync() is a synchronous method that assumes the document is already available. +The examples below illustrate asynchronously loading a document or synchronously loading a document and then interacting with it:

> const handle = repo.find("automerge:2j9knpCseyhnK8izDmLpGP5WMdZQ")
> const doc = await handle.doc()
> console.log(doc)

Or 

> const handle = repo.find("automerge:2j9knpCseyhnK8izDmLpGP5WMdZQ")
> handle.whenReady().then(() => {
  console.log(handle.docSync())
})

In this latter example we use DocHandle.whenReady, which returns a promise that the repository resolves when it loads a document from storage or fetches it from another peer in the network.

change() and on("change")​

Use DocHandle.change when you modify a document.

> const handle = repo.find("automerge:2j9knpCseyhnK8izDmLpGP5WMdZQ")
> await handle.doc()
> handle.change(d => d.foo = "bar")

The Repo calls DocHandle.on("change") whenever the document is modified – either due to a local change or a sync message being received from another peer.

> const handle = repo.find("automerge:4CkUej7mAYnaFMfVnffDipc4Mtvn")
> await handle.doc()
> handle.on("change", ({doc}) => {
  console.log("document changed")
  console.log("New content: ", doc)
})

Integrations​

automerge-repo provides a set of primitives that you can use to build a wide range of applications. To make this easier, we have built integrations with a few common UI frameworks. You can easily add further integrations and we welcome contributions which integrate with popular frameworks!

React Integration​

@automerge/automerge-repo-react-hooks makes it easy to use automerge-repo in a React application. Once you've constructed a Repo you can make it available to your React application using RepoContext. Once available, call useHandle to obtain a DocHandle:

function TodoList(listUrl: AutomergeUrl) {
    const handle = useHandle(listUrl)
    // render the todolist
}

Note that when Repo receives changes over the network or registers local changes, the original Automerge document remains immutable, and any modified parts of the document get new objects. This means that React will only re-render the parts of the UI that depend on a part of the document that has changed.

Svelte Integration​

@automerge/automerge-repo-svelte-store provides setContextRepo to set the Repo which is used by the document store:

<script lang="ts">
  import { document } from "@automerge/automerge-repo-svelte-store"
  import { type AutomergeUrl } from "@automerge/automerge-repo"

  export let documentUrl: AutomergeUrl

  // Doc is an automerge store with a `change` method which accepts
  // a standard automerge change function
  const doc = document<HasCount>(documentUrl)
  const increment = () => {
    doc.change((d: HasCount) => (d.count = (d.count || 0) + 1))
  }
</script>

<button on:click={increment}>
  count is {$doc?.count || 0}
</button>

What about <X>?​

We'd love to help you make automerge work in your favorite development environment! Please reach out to us on GitHub or via our Slack.

Extending automerge-repo​

You can extend automerge-repo by writing new storage or network adapters.

Storage Adapters​

A storage adapter represents some kind of backend that stores the data in a repo. Storage adapters can be implemented for any key/value store that allows you to query a range of keys with a given prefix. There is no concurrency control required (that's implemented in automerge-repo) so you can safely have multiple repos pointing at the same storage. For example, you could implement an adapter on top of Redis.

The automerge-repo library provides storage adapters for IndexedDB and the file system (on Node).

Network Adapters​

A network adapter represents a way of connecting to other peers. Network adapters raise events when a new peer is discovered or when a message is recieved, and implement a send method for transmitting messages to another peer. automerge-repo assumes a reliable, in-order transport for each peer; as long as you can provide this (e.g. using a TCP connection), you can implement an adapter. You could implement an adapter for BLE, for example.

The automerge-repo library provides network adapters for WebSocket, MessageChannel, and BroadcastChannel.

Other languages/platforms​

This release of automerge-repo is just for javascript. Automerge is a multi-language library though and there are efforts under way to implement automerge-repo on other platforms. The most mature of these is automerge-repo-rs. We welcome contributions and please reach out if you're starting to develop automerge-repo for a new platform.

Beta Quality​

automerge-repo works pretty well – we're using it at Ink & Switch for a bunch of internal projects. The basic shape of the API is simple and useful, and not having to think about the plumbing makes it much, much faster to get a useful application off the ground. However, there are some performance problems we're working on:

  1. Documents with large histories (e.g. a collaboratively edited document with >60,000 edits) can be slow to sync.
  2. The sync protocol currently requires that a document it is syncing be loaded into memory. This means that a sync server can struggle to handle a lot of traffic on large documents.

These two points mean that we're not ready to say this project is ready for production.

We're working hard on fixing the performance so that we can say this is ready for production. But if you are interested in experimenting with the library now, or if you are only going to be working with relatively small documents or low traffic sync servers then you are good to go!

(If you want us to get to production faster, or you have some specific requirements, please consider sponsoring Automerge development 🙂)

Finally, we don't want to give the impression that everything is smooth sailing. automerge-repo solves a bunch of the hard problems people were encountering around networking and storage. There are still plenty of other difficult problems in local first software where we don't have turnkey solutions: authentication and authorization, end-to-end encryption, schema changes, version control workflows etc. automerge-repo makes many things much easier, but it's a frontier out here.

]]> + <![CDATA[Automerge 2.0]]> https://automerge.github.io/blog/automerge-2 diff --git a/blog/tags/hello/index.html b/blog/tags/hello/index.html index 08ae1e97..71a12bc1 100644 --- a/blog/tags/hello/index.html +++ b/blog/tags/hello/index.html @@ -10,13 +10,13 @@ - - + +
-

One post tagged with "hello"

View All Tags

· One min read
Rae McKelvey

You've reached the Automerge docs! We're so happy to have you.

We're using Docusaurus. Please help edit the docs on GitHub.

- - +

One post tagged with "hello"

View All Tags

· One min read
Rae McKelvey

You've reached the Automerge docs! We're so happy to have you.

We're using Docusaurus. Please help edit the docs on GitHub.

+ + \ No newline at end of file diff --git a/blog/tags/index.html b/blog/tags/index.html index 82b10de7..ad234af8 100644 --- a/blog/tags/index.html +++ b/blog/tags/index.html @@ -10,13 +10,13 @@ - - + +
-

Tags

- - +

Tags

+ + \ No newline at end of file diff --git a/blog/welcome/index.html b/blog/welcome/index.html index 4f1dd327..6e80201f 100644 --- a/blog/welcome/index.html +++ b/blog/welcome/index.html @@ -10,13 +10,13 @@ - - + +
-

Welcome

· One min read
Rae McKelvey

You've reached the Automerge docs! We're so happy to have you.

We're using Docusaurus. Please help edit the docs on GitHub.

- - +

Welcome

· One min read
Rae McKelvey

You've reached the Automerge docs! We're so happy to have you.

We're using Docusaurus. Please help edit the docs on GitHub.

+ + \ No newline at end of file diff --git a/docs/api/index.html b/docs/api/index.html index 7d4425bc..694626d6 100644 --- a/docs/api/index.html +++ b/docs/api/index.html @@ -10,13 +10,13 @@ - - + +
- - + + \ No newline at end of file diff --git a/docs/concepts/index.html b/docs/concepts/index.html index ed70cb3a..8b638b3f 100644 --- a/docs/concepts/index.html +++ b/docs/concepts/index.html @@ -10,13 +10,13 @@ - - + +

Concepts

info

This documentation is mostly focused on the javascript implementation of automerge. Some things will translate to other languages but some things - in particular the "repository" concept and automerge-repo library - will not.

Core concepts​

Using automerge means storing your data in automerge documents. Documents have a URLs which you can use to share or request documents with/from other peers using a repository. Repositories give you DocHandles which you use to make changes to the document and listen for changes from other peers.

Automerge as used in javascript applications is actually a composition of two libraries. automerge-repo which provides the networking and storage plumbing, and automerge which provides the CRDT implementation, a transport agnostic sync protocol, and a compressed storage format which automerge-repo uses to implement various networking and storage plugins.

Documents​

A document is the "unit of change" in automerge. It's like a combination of a JSON object and a git repository. What does that mean?

Like a JSON object an automerge document is a map from strings to values, where the values can themselves be maps, arrays, or simple types like strings or numbers. See the data model section for more details.

Like a git repository an automerge document has a history made up of commits. Every time you make a change to a document you are adding to the history of the document. The combination of this history and some rules about how to handle conflicts means that any two automerge documents can always be merged. See merging for the gory details.

Repositories​

A repository manages connections to remote peers and access to some kind of local storage. Typically you create a repository at application startup and then inject it into the parts of your application which need it. The repository gives out DocHandles, which allow you to access the current state of a document and make changes to it without thinking about how to store those changes, transmit them to others, or fetch changes from others.

Networking and storage for a repository is pluggable. There are various ready made network transports and storage implementations but it is also easy to build your own.

DocHandles​

A DocHandle is an object returned from the various methods on a repository which create or request a document. The DocHandle has methods on it to access the underlying automerge document and to create new changes which are stored locally and transmitted to connected peers.

Document URLs​

Documents in a repository have a URL. An automerge URL looks like this:

automerge:2akvofn6L1o4RMUEMQi7qzwRjKWZ

That is, a string of the form automerge:<base58>. This URL can be passed to a repository which will use it to check if the document is in any local storage or available from any connected peers.

Sync Protocol​

Repositories communicate with each other using an efficient sync protocol which is implmented in automerge. This protocol is transport agnostic and works on a per-document basis, a lot of the work automerge-repo does is handling running this sync protocol for multiple documents over different kinds of network.

Storage Format​

automerge implements a compact binary storage format which makes it feasible to store all the editing history of a document (for example, storing every keystroke in a large text document). automerge-repo implements the common logic of figuring out when to compress documents and doing so in a way which is safe for concurrent reads and writes.

- - + + \ No newline at end of file diff --git a/docs/cookbook/modeling-data/index.html b/docs/cookbook/modeling-data/index.html index 21757ee1..5b057afc 100644 --- a/docs/cookbook/modeling-data/index.html +++ b/docs/cookbook/modeling-data/index.html @@ -10,13 +10,13 @@ - - + +

Modeling Data

All data in Automerge must be stored in a document. A document can be modeled in a variety of ways, and there are many design patterns that can be used. An application could have many documents, typically identified by a UUID.

In this section, we will discuss how to model data within a particular document, including how to version and manage data with Automerge in production scenarios.

How many documents?​

You can decide which things to group together as one Automerge document (more fine grained or more coarse grained) based on what makes sense in your app. Having hundreds of docs should be fine — we've built prototypes of that scale. One major automerge project, PushPin, was built around very granular documents. This had a lot of benefits, but the overhead of syncing many thousands of documents was high.

We believe on the whole there's an art to the granularity of data that is universal. When should you have two JSON documents or two SQLite databases or two rows? We suspect that an Automerge document is best suited to being a unit of collaboration between two people or a small group.

Setting up an initial document structure​

When you create a document using Automerge.init(), it's just an empty JSON document with no properties. As the first change, most applications will need to initialize some empty collection objects that are expected to be present within the document.

The easiest way of doing this is with a call to Automerge.change() that sets up the document schema in the form that you need it, like in the code sample above. You can then sync this initial change to all of your devices; once everybody has the schema, you can have different users updating the document on different devices, and the updates should merge nicely. For example:

// Set up the `cards` array in doc1
let doc1 = Automerge.change(Automerge.init(), doc => {
  doc.cards = []
})

// In doc2, don't create `cards` again! Instead, merge
// the schema initialization from doc1
let doc2 = Automerge.merge(Automerge.init(), doc1)

// Now we can update both documents
doc1 = Automerge.change(doc1, doc => {
  doc.cards.push({ title: 'card1' })
})

doc2 = Automerge.change(doc2, doc => {
  doc.cards.push({ title: 'card2' })
})

// The merged document will contain both cards
doc1 = Automerge.merge(doc1, doc2)
doc2 = Automerge.merge(doc2, doc1)

However, sometimes it's inconvenient to have to sync the initial change to a device before you can modify the document on that device. If you want two devices to be able to independently set up their own document schema, but still to be able to merge those documents, you have to be careful. Simply doing Automerge.change() on each device to initialize the schema will not work, because you now have two different documents with no shared ancestry (even if the initial change performs the same operations, each device has a different actorId and so the changes will be different).

If you really must initialize each device's copy of a document independently, one option is to do the initial Automerge.change() once to set up your schema, then call Automerge.save() on the document (which returns a byte array), and hard-code that byte array into your application. Now, on each device that needs to initialize a document, you do this:

// hard-code the initial change here
const initChange = new Uint8Array([133, 111, 74, 131, ...])
let [doc] = Automerge.load(initChange)

This will set you up with a document whose initial change is the one you hard-coded. Any documents you set up with the same initial change will be able to merge.

Versioning​

Often, there comes a time in the production lifecycle where you will need to change the schema of a document. Because Automerge uses a JSON document model, it's similar to a NoSQL database, where properties can be arbitrarily removed and added at will.

You can implement your own versioning scheme, for example by embedding a schema version number into the document, and writing a function that can upgrade a document from one schema version to the next. However, doing this in a CRDT like Automerge is more difficult than migrations in a centralized relational database, because it could happen that two users independently perform the same migration. In this case, you need to ensure that the two migrations don't clash with each other, which is difficult.

One way of making migrations safe is by using the tricks from the previous section: in addition to hard-coding the initial change that sets up the document, you can also hard-code migrations that upgrade from one schema version to the next, using the same technique (either hard-coding the change as a byte array, or making a change on the fly with hard-coded actorId and timestamp). Do not modify the initial change; instead, every migration should be a separate hard-coded change that depends only on the preceding change. This way, you can have multiple devices independently applying the same migration, and they will all be compatible because the migration is performed identically on every device.

type DocV1 = { 
  version: 1,
  cards: Card[]
}

type DocV2 = { 
  version: 2,
  title: Automerge.Text,
  cards: Card[]
}

// This change creates the `title` property required in V2,
// and updates the `version` property from 1 to 2
const migrateV1toV2 = new Uint8Array([133, 111, 74, 131, ...])

let doc = getDocumentFromNetwork()
if (doc.version === 1) {
  [doc] = Automerge.applyChange(doc, [migrateV1toV2])
}

Also keep in mind that in your app there might be some users using an old version of the app while other users are using a newer version; you will need to take care with migrations to ensure that they do not break compatibility with older app versions, or force all users to update to the latest version.

Some further ideas on safe schema migrations in CRDT apps are discussed in the Cambria paper, but these are not yet implemented in Automerge. If you want to work on improving schema migrations in Automerge, please get in touch — contributions are welcome!

Performance​

Automerge documents hold their entire change histories. It is fairly performant, and can handle a significant amount of data in a single document's history. Performance depends very much on your workload, so we strongly suggest you do your own measurements with the type and quantity of data that you will have in your app.

Some developers have proposed “garbage collecting” large documents. If a document gets to a certain size, a central authority could emit a message to each peer that it would like to reduce it in size and only save the history from a specific change (hash). Martin Kleppmann did some experiments with a benchmark document to see how much space would be saved by discarding history, with and without preserving tombstones. See this video at 55 minutes in. The savings are not all that great, which is why we haven't prioritised history truncation so far.

Typically, performance improvements can come at the networking level. You can set up a single connection (between peers or client-server) and sync many docs over a single connection. The basic idea is to tag each message with the ID of the document it belongs to. There are possible ways of optimising this if necessary. In general, having fewer documents that a client must load over the network or into memory at any given time will reduce the synchronization and startup time for your application.

- - + + \ No newline at end of file diff --git a/docs/documents/conflicts/index.html b/docs/documents/conflicts/index.html index 2c1c3acb..5b31bb95 100644 --- a/docs/documents/conflicts/index.html +++ b/docs/documents/conflicts/index.html @@ -10,8 +10,8 @@ - - + +
@@ -30,7 +30,7 @@ conflicts object to show the conflict in the user interface. The keys in the conflicts object are the internal IDs of the operations that updated the property x.

The next time you assign to a conflicting property, the conflict is automatically considered to be resolved, and the conflict disappears from the object returned by Automerge.getConflicts().

Automerge uses a combination of LWW (last writer wins) and multi-value register. By default, if you read from doc.foo you will get the LWW semantics, but you can also see the conflicts by calling Automerge.getConflicts(doc, 'foo') which has multi-value semantics.

Every operation has a unique operation ID that is the combination of a counter and the actorId that generated it. Conflicts are ordered based on the counter first (using the actorId only to break ties when operations have the same counter value).

- - + + \ No newline at end of file diff --git a/docs/documents/counters/index.html b/docs/documents/counters/index.html index bfe8e906..0246d702 100644 --- a/docs/documents/counters/index.html +++ b/docs/documents/counters/index.html @@ -10,8 +10,8 @@ - - + +
@@ -25,7 +25,7 @@ decrease the counter value, you can use the .increment() or .decrement() method:

state = Automerge.change(state, doc => {
  doc.buttonClicks.increment() // Add 1 to counter value
  doc.buttonClicks.increment(4) // Add 4 to counter value
  doc.buttonClicks.decrement(3) // Subtract 3 from counter value
})

Note: In relational databases it is common to use an auto-incrementing counter to generate primary keys for rows in a table, but this is not safe in Automerge, since several users may end up generating the same counter value! Instead it is best to use UUIDs to identify entities.

- - + + \ No newline at end of file diff --git a/docs/documents/index.html b/docs/documents/index.html index 1a0a0a27..a67c3410 100644 --- a/docs/documents/index.html +++ b/docs/documents/index.html @@ -10,13 +10,13 @@ - - + +

Document Data Model

Automerge documents are quite similar to JSON objects. A document always consists of a root map which is a map from strings to other automerge values, which can themselves be composite types.

The types in automerge are:

  • Composite types
  • Scalar (non-composite) types:
    • IEEE 754 64 bit floating point numbers
    • Unsigned integers
    • Signed integers
    • Booleans Strings
    • Timestamps
    • Counters
    • Byte arrays

See below for how these types map to JavaScript types.

Maps​

Maps have string keys and any automerge type as a value. "string" here means a unicode string. The underlying representation in automerge is as UTF-8 byte sequences but they are exposed as utf-16 strings in javascript.

Lists​

A list is an ordered sequence of automerge values. The underlying data structure is an RGA sequence, which means that concurrent insertions and deletions can be merged in a manner which attempts to preserve user intent.

Text​

Text is an implementation of the peritext CRDT. This is conceptually similar to a list where each element is a single unicode scalar value representing a single character. In addition to the characters Text also supports "marks". Marks are tuples of the form (start, end, name, value) which have the following meanings:

  • start - the index of the beginning of the mark
  • end - the index of the end of the mark
  • name - the name of the mark
  • value - any scalar (as in automerge scalar) value

For example, a bold mark from charaters 1 to 5 might be represented as (1, 5, "bold", true).

Note that the restriction to scalar values for the value of a mark will be lifted in future, although mark values will never be mutable - instead you should always create a new mark when updating a value. For now, if you need complex values in a mark you should serialize the value to a string.

Timestamps​

Timestamps are the integer number of milliseconds since the unix epoch (midnight 1970, UTC).

Counter​

Counters are a simple CRDT which just merges by adding all concurrent operations. They can be incremented and decremented.

Javascript language mapping​

The mapping to javascript is accomplished with the use of proxies. This means that in the javascript library maps appear as objects and lists appear as Arrays. There is only one numeric type in javascript - number - so the javascript library guesses a bit. If you insert a javascript number for which Number.isInteger returns true then the number will be inserted as an integer, otherwise it will be a floating point value.

How Text and String are represented will depend on whether you are using the next API

Timestamps are represented as javascript Dates.

Counters are represented as instances of the Counter class.

Putting it all together, here's an example of an automerge document containing all the value types:

import  * as A  from "@automerge/automerge/next"

let doc = A.from({
    map: { 
        key: "value",
        nested_map: {key: "value"},
        nested_list: [1]
    },
    list: ["a", "b", "c", {nested: "map"}, ["nested list"]],
    // Note we are using the `next` API for text, so text sequences are strings
    text: "some text",
    // In the `next` API non mergable strings are instances of `RawString`.
    // You should generally not need to use these. They are retained for backwards
    // compatibility
    raw_string: new A.RawString("rawstring"), 
    integer: 1,
    float: 2.3,
    boolean: true,
    bytes: new Uint8Array([1, 2, 3]),
    date: new Date(),
    counter: new A.Counter(1),
    none: null,
})

doc = A.change(doc, d => {
    // Insert 'Hello' at the begnning of the string
    A.splice(d, ["text"], 0, 0, "Hello ")
    d.counter.increment(20)
    d.map.key = "new value"
    d.map.nested_map.key = "new nested value"
    d.list[0] = "A"
    d.list.insertAt(0, "Z")
    d.list[4].nested = "MAP"
    d.list[5][0] = "NESTED LIST"
})

console.log(doc)

// Prints
// {
//   map: {
//     key: 'new value',
//     nested_map: { key: 'new nested value' },
//     nested_list: [ 1 ]
//   },
//   list: [ 'Z', 'A', 'b', 'c', { nested: 'MAP' }, [ 'NESTED LIST' ] ],
//   text: 'Hello world',
//   raw_string: RawString { val: 'rawstring' },
//   integer: 1,
//   float: 2.3,
//   boolean: true,
//   bytes: Uint8Array(3) [ 1, 2, 3 ],
//   date: 2023-09-11T13:35:12.229Z,
//   counter: Counter { value: 21 },
//   none: null
// }
- - + + \ No newline at end of file diff --git a/docs/documents/lists/index.html b/docs/documents/lists/index.html index 77514676..4fea5d8e 100644 --- a/docs/documents/lists/index.html +++ b/docs/documents/lists/index.html @@ -10,8 +10,8 @@ - - + +
@@ -29,7 +29,7 @@ much better with concurrent updates by different users.

As a general principle with Automerge, you should make state updates at the most fine-grained level possible. Don't replace an entire object if you're only modifying one property of that object; just assign that one property instead.

- - + + \ No newline at end of file diff --git a/docs/documents/text/index.html b/docs/documents/text/index.html index a8ee4ba7..174b49e3 100644 --- a/docs/documents/text/index.html +++ b/docs/documents/text/index.html @@ -10,8 +10,8 @@ - - + +
@@ -21,7 +21,7 @@ deleteAt() to insert and delete characters (same API as for list modifications, shown above):

newDoc = Automerge.change(currentDoc, doc => {
  doc.text = new Automerge.Text()
  doc.text.insertAt(0, 'h', 'e', 'l', 'l', 'o')
  doc.text.deleteAt(0)
  doc.text.insertAt(0, 'H')
})

To inspect a text object and render it, you can use the following methods (outside of a change callback):

newDoc.text.length // returns 5, the number of characters
newDoc.text.get(0) // returns 'H', the 0th character in the text
newDoc.text.toString() // returns 'Hello', the concatenation of all characters
for (let char of newDoc.text) console.log(char) // iterates over all characters

To figure out which regions were inserted by which users, you can use the elementId. The ElementID gives is a string of the form ${actorId}@${counter}. Here, actorId is the ID of the actor who inserted that character.

let elementId = newDoc.text.getElemId(index)
// '2@369125d35a934292b6acb580e31f3613'

Note that the actorId changes with each call to Automerge.init().

- - + + \ No newline at end of file diff --git a/docs/documents/values/index.html b/docs/documents/values/index.html index 5687e09e..d7605849 100644 --- a/docs/documents/values/index.html +++ b/docs/documents/values/index.html @@ -10,13 +10,13 @@ - - + +

Simple Values

All JSON primitive datatypes are supported in an Automerge document. In addition, JavaScript Date objects are supported.

Remember, never modify currentDoc directly, only ever change doc inside the callback to Automerge.change!

newDoc = Automerge.change(currentDoc, doc => {
  doc.property = 'value' // assigns a string value to a property
  doc['property'] = 'value' // equivalent to the previous line
  delete doc['property'] // removes a property

  doc.stringValue = 'value'
  doc.numberValue = 1
  doc.boolValue = true
  doc.nullValue = null
  doc.nestedObject = {} // creates a nested object
  doc.nestedObject.property = 'value'
  // you can also assign an object that already has some properties
  doc.otherObject = { key: 'value', number: 42 }
})
- - + + \ No newline at end of file diff --git a/docs/glossary/index.html b/docs/glossary/index.html index 788b0bd6..29c709a7 100644 --- a/docs/glossary/index.html +++ b/docs/glossary/index.html @@ -10,13 +10,13 @@ - - + +

Glossary

CRDTs​

Automerge is a type of CRDT (Conflict-Free Replicated Datatype). A CRDT is a data structure that simplifies multi-user applications. We can use them to synchronize data between two devices in a way that both devices see the same application state. In many systems, copies of some data need to be stored on multiple computers. Examples include:

  • Mobile apps that store data on the local device, and that need to sync that data to other devices belonging to the same user (such as calendars, notes, contacts, or reminders);
  • Distributed databases, which maintain multiple replicas of the data (in the same datacenter or in different locations) so that the system continues working correctly if some of the replicas are offline;
  • Collaboration software, such as Google Docs, Trello, Figma, or many others, in which several users can concurrently make changes to the same file or data;
  • Large-scale data storage and processing systems, which replicate data in order to achieve global scalability.

Read more about CRDTs

Eventual Consistency​

Applications built with Automerge are eventually consistent. This means if several users are working together, they will eventually all see the same application state, but at any given moment it's possible for the users to be temporarily out of sync.

Eventual consistency allows applications to work offline: even if a user is disconnected from the internet, Automerge allows that user to view and modify their data. If the data is shared between several users, they may all update their data independently. Later, when a network is available again, Automerge ensures that those edits are cleanly merged. See the page on conflicts for more detail on these merges.

Documents​

A document is a collection of data that holds the current state of the application. A document in Automerge is represented as an object. Each document has a set of keys which can be used to hold variables that are one of the Automerge datatypes.

Types​

All collaborative data structures conform to certain rules. Each variable in the document must be of one of the implemented types. Each type must conform to the rules of CRDTs. Automerge comes with a set of pre-defined types such as Map, Array, Counter, number, Text, and so on.

Changes​

A change describes some update to a document; think of it like a commit in Git. A change could perform several operations, for example setting several properties or updating several objects within the document, and these will all be executed atomically. Changes are commutative, which means that the order in which they are applied does not matter. When the same set of changes has been applied to two documents, Automerge guarantees that they will be in the same state.

To do this, typically each change depends upon a previous change. Automerge creates a directed acyclic graph (DAG) of changes. To learn more about how automerge works internally, see the Internals section.

History​

Each change that is made to a data structure builds upon other changes to create a shared, materialized view of a document. Each change is dependent on a previous change, which means that all replicas are able to construct a history of the data structure. This is a powerful property in multi-user applications, and can be implemented in a way that is storage and space efficient.

Compaction​

Compaction is a way to serialize the current state of the document without the history. You might want to do this when:

  • You don't want to replicate the entire history because of bandwidth or resource concerns on the target device. This might be useful in embedded systems or mobile phones.
  • A deleted element contains some sensitive information that you would like to be purged from the history.

The downsides of compacting the history of a document include not being able to synchronize that compacted document with another document that doesn't have a common ancestor.

Synchronization​

When two or more devices make changes to a document, and then decide to exchange those changes to come to a consistent state, we call that synchronization. Synchronization can, in the most simple implementation, consist of sending the full list of changes in the history to all connected devices. To improve performance, devices may negotiate which changes are missing on either end and exchange only those changes which are missing, rather than the entire change history.

- - + + \ No newline at end of file diff --git a/docs/hello/index.html b/docs/hello/index.html index 24725bf4..7a7e7135 100644 --- a/docs/hello/index.html +++ b/docs/hello/index.html @@ -10,8 +10,8 @@ - - + +
@@ -52,7 +52,7 @@ useful benefits, such as allowing several clients to concurrently update the data, easy sync between clients and server, being able to inspect the change history of your app's data, and support for branching and merging workflows.

- - + + \ No newline at end of file diff --git a/docs/quickstart/index.html b/docs/quickstart/index.html index 63dac332..c646997e 100644 --- a/docs/quickstart/index.html +++ b/docs/quickstart/index.html @@ -10,13 +10,13 @@ - - + +

5-Minute Quick Start

It's easy to build a local-first web application with real-time synchronization using Automerge. In this quickstart, we'll start with the standard yarn create vite example Typescript application and use Automerge to turn it into a simple local-first application.

All the code here can be found at the automerge-repo-quickstart repo.

Let's begin.

Setup​

First, let's initialize an off-the-shelf React app using Vite as our bundler. We're not going to remind you along the way, but we recommend you initialize a git repo and check in the code at whatever interval feels comfortable.

$ yarn create vite
# Project name: hello-automerge-repo
# Select a framework: React
# Select a variant: TypeScript

$ cd hello-automerge-repo
$ yarn

Next, we'll add some automerge dependencies for the project. We'll introduce each of these libraries as they come up in the tutorial.

yarn add @automerge/automerge \
  @automerge/automerge-repo \
  @automerge/automerge-repo-react-hooks \
  @automerge/automerge-repo-network-broadcastchannel \
  @automerge/automerge-repo-storage-indexeddb \
  vite-plugin-wasm \
  vite-plugin-top-level-await

Because Vite support for WebAssembly modules (used by Automerge) currently requires configuring a plugin, replace vite.config.ts with the following:

// vite.config.ts
import { defineConfig } from "vite"
import react from "@vitejs/plugin-react"
import wasm from "vite-plugin-wasm"
import topLevelAwait from "vite-plugin-top-level-await"

export default defineConfig({
  plugins: [topLevelAwait(), wasm(), react()],

  worker: {
    format: "es",
    plugins: [wasm()],
  },
})

With that out of the way, we're ready to build the application.

Using Automerge

The central concept of Automerge is one of documents. An Automerge document is a JSON-like data structure that is kept synchronized between all communicating peers with the same document ID.

To create or find Automerge documents, we'll use a Repo. The Repo (short for repository) keeps track of all the documents you load and makes sure they're properly synchronized and stored. Let's go ahead and make one. Add the following imports to src/main.tsx:

import { isValidAutomergeUrl, Repo } from '@automerge/automerge-repo'
import { BroadcastChannelNetworkAdapter } from '@automerge/automerge-repo-network-broadcastchannel'
import { IndexedDBStorageAdapter } from "@automerge/automerge-repo-storage-indexeddb"
import {next as A} from "@automerge/automerge" //why `next`? See the the "next" section of the conceptual overview

Initializing a repository​

Before we can start finding or creating documents, we'll need a repo. Here, we create one that can synchronize with other tabs using a sort of pseudo-network built into the browser that allows communication between tabs with the same shared origin: the BroadcastChannel.

const repo = new Repo({
  network: [new BroadcastChannelNetworkAdapter()],
  storage: new IndexedDBStorageAdapter(),
})

Creating (or finding) a document​

Now that we have the repo, we want to either create a document if we don't have one already or we want to load a document. To keep things simple, we'll check the URL hash for a document ID, and if we don't find one, we'll start a new document and set it in the hash.

Add this code right after the repo initialization code.

const rootDocUrl = `${document.location.hash.substr(1)}`
let handle
if (isValidAutomergeUrl(rootDocUrl)) {
    handle = repo.find(rootDocUrl)
} else {
    handle = repo.create<{counter?: A.Counter}>()
    handle.change(d => d.counter = new A.Counter())
}
const docUrl = document.location.hash = handle.url
// @ts-ignore
window.handle = handle // we'll use this later for experimentation

A real application would probably handle routing differently, but this is enough to get started.

Working with the document​

The main way of interacting with a Repo is through DocHandles, which allow you to read data from a document or make changes to it and which emit "change" events whenever the document changes -- either through local actions or over the network.

Now that we have a document handle stuck onto the window, let's experiment with it. Start your application now with:

$ yarn dev

You won't see any changes from the default example application on screen, but we've attached an Automerge document to the window object, which makes it conveniently available in the Chrome debugger.

Your new document is empty, because we just created it. Let's start by initializing a counter. Run the following command in your Chrome debugger.

handle.change(d => { d.counter.increment(10) })

DocHandle.change allows you to modify the document managed by a DocHandle and takes care of storing new changes and notifying any peers of new changes.

Next, run this code to see the contents of your document. The contents will look a bit complex, but you should see a counter with a value of 10 if you poke around.

handle.docSync()

Calling DocHandle.docSync() return the current value of the document synchronously, or returns undefined if the document is unavailable either because it is still loading, or because it can't be found. To avoid this problem, prefer the asynchronous form: await handle.doc(). If you want to render loading states differently from an unavailable state, you can inspect handle.state and branch accordingly.

Updating your app to use Automerge​

We've already created or fetched our initial document via main.tsx, but usually when when we want to work with a document in a React application, we will refer to it by URL. Let's start by editing the call signature for App.tsx to pass in the URL for your newly created document, and then make it available to your component with the useDocument hook.

We also need to make the repo object we created available throughout the application, so we use a React Context provider for that. In main.tsx, modify the React.render() call to look like this:

ReactDOM.createRoot(document.getElementById('root')!).render(
  <React.StrictMode>
    <RepoContext.Provider value={repo}>
      <App docUrl={docUrl}/>
    </RepoContext.Provider>
  </React.StrictMode>,
)

and also add another import line:

import { RepoContext } from '@automerge/automerge-repo-react-hooks'

Inside App.tsx, add these imports:

import {AutomergeUrl} from '@automerge/automerge-repo'
import {useDocument} from '@automerge/automerge-repo-react-hooks'
import {next as A} from "@automerge/automerge"

and change the first few lines to these:

interface CounterDoc {
  counter: A.Counter
}

function App({docUrl}: {docUrl: AutomergeUrl}) {
  const [doc, changeDoc] = useDocument<CounterDoc>(docUrl)

Now you've got access to the document in a more native React-style way: a hook that will update every time the document changes.

Our last step here is to change our code to use these new values by replacing how we render the button element.

        <button onClick={() => changeDoc((d) => d.counter.increment(1))}>
          count is { doc && doc.counter.value }
        </button>

Go ahead and try this out. Open a second (or third) tab with the same URL and see how as you click the counter in any tab, the others update.

If you close all the tabs and reopen them, the counter value is preserved.

Congratulations! You have a working Automerge-backed React app with live local synchronization. How does it work? We'll learn through some experimentation in the next section.

Collaborating over the internet​

The handle we have created has a URL, we can access that with DocHandle.url, this URL can be used to sync the document with any peer who has it. Open up your browser debugger and run console.log(handle.url), this should print something that looks like "automerge:45NuQi1e45PKsemx8GhSCu62gyag", make a note of this for later.

First, we'll add a network adapter to the Repo in our web app which syncs to a sync server via a websocket. Add the following dependency to the web app we've been building:

yarn add @automerge/automerge-repo-network-websocket

Then add a network adapter connecting the repo to sync.automerge.org

// main.tsx
// Add this import
import { BrowserWebSocketClientAdapter } from "@automerge/automerge-repo-network-websocket"

...

// now update the repo definition to look like this:
const repo = new Repo({
    network: [
        new BroadcastChannelNetworkAdapter(),
        // This is the new line
        new BrowserWebSocketClientAdapter('wss://sync.automerge.org')
    ],
    storage: new IndexedDBStorageAdapter(),
})

This creates a repo which syncs changes it sees to sync.automerge.org and any other process can connect to that server and use the URL to get the changes we've made.

note

The Automerge project provides a public sync server for you to experiment with sync.automerge.org. This is not a private instance, and as an experimental service has no reliability or data safety guarantees. Basically, it's good for demos and prototyping, but run your own sync server for production uses.

To see this in action we'll create a little node app. Change into a clean directory and run

npm create @automerge/repo-node-app amg-quickstart
cd amg-quickstart

Now open index.js and add the following:

// repo is already set up by the `repo-node-app` helper
const doc = repo.find("<url copied from the debugger>")
console.log(await doc.doc())
// This is required because we don't have a way of shutting down the repo
setTimeout(() => process.exit(), 1000)

Now run this with node index.js and you should see the contents of the document.

Now add the following at the end of index.js (but before the setTimeout)

doc.change(d => {
    d.counter.increment(1)
})

This change will be reflected in any connected and listening handles. Go back to the original browser window and watch it as you run node index.js. What you should see is that every time you run the script the counter in the browser changes.

Saving the document​

If you provide a Repo with a StorageAdapter then it will save documents for use later. In the browser we might used IndexedDB:

import { IndexedDBStorageAdapter } from "@automerge/automerge-repo-storage-indexeddb"
import { BrowserWebSocketClientAdapter } from "@automerge/automerge-repo-network-websocket"

const repo = new AutomergeRepo.Repo({
  network: [new BrowserWebSocketClientAdapter("wss://sync.automerge.org")],
  storage: new IndexedDBStorageAdapter(),
})

Documents will be stored in IndexedDB and methods like Repo.find will consult storage when loading. The upshot is that if you had a document locally, it will continue to be available regardless of whether you are connected to any peers.

More​

If you're hungry for more, look in the Cookbook section.

- - + + \ No newline at end of file diff --git a/docs/repositories/dochandles/index.html b/docs/repositories/dochandles/index.html index 3d72fc91..ac2bb6f9 100644 --- a/docs/repositories/dochandles/index.html +++ b/docs/repositories/dochandles/index.html @@ -10,13 +10,13 @@ - - + +

DocHandles

Once you have a Repo with a NetworkAdapter and a StorageAdapter you can get down to the business of creating and working with DocHandles.

It's useful to understand a little about why we need a DocHandle. @automerge/automerge documents are fairly inert data structures. You can create a document, you can mutate it, you can generate sync messages to send elsewhere and you can receive sync messages from elsewhere. None of this is very "live" though. Because the document has no concept of a network, or of storage, you can't say "every time I change a document, tell everyone else about it and save the change to storage". This "live document" is what a DocHandle is. A DocHandle is a wrapper around a document managed by a Repo. It provides the following kinds of "liveness":

  • Whenever you change the document using DocHandle.change or DocHandle.changeAt the changes will be saved to the attached StorageAdapter and sent to any connected NetworkAdapters
  • Whenever a change is received from a connected peer the DocHandle will fire a "change" event
  • There is a concept of an ephemeral message, which you can send using DocHandle.broadcast. Whenever a DocHandle receives an ephemeral message it will fire a "ephemeral-message" event
  • You can wait for a DocHandle to be loaded, or to be retrieved from another peer
  • DocHandles have a URL, which can be used to uniquely refer to the document it wraps when requesting it from another peer

DocHandles are very useful, how do you obtain one?

Creating a DocHandle​

This is the easy one, just call Repo.create. This creates a new document, stores it, and then enqueues messages to all connected peers informing them of the new document.

Waiting for a DocHandle​

Typically you are not creating a new document, but working with an existing one. Maybe the document URL was stored in localStorage, maybe the URL was in the hash fragment of the browser, etc. In this case you use Repo.find to lookup the document. This means the DocHandle can be in several different states, to understand this we'll first look at the states in detail, then some convenience methods DocHandle exposes for waiting for different states.

DocHandle states​

Repo.find will do two things simultaneously:

  • Look in the attached StorageAdapter to see if we have any data for the document
  • Send a request to any connected peers to ask if they have the document

These actions are asynchronous, as they complete the state of the document changes. This state is represented most explicitly in the HandleState enum, which has the following states:

  • IDLE - This is really just a start state, every dochandle immediately transitions to another state
  • AWAITING_NETWORK - in this state we are waiting for the NetworkAdapters to be ready to process messages. This typically occurs at application startup. Most NetworkAdapters have an asynchronous startup period. The Repo waits until every NetworkAdapter has emitted a ready event before beginning to request documents
  • LOADING - we are waiting for storage to finish trying to load this document
  • REQUESTING - we are waiting to hear back from other peers about this document
  • READY - The document is available, either we created it, found it in storage, or someone sent it to us
  • DELETED - The document was removed from the repo
  • UNAVAILABLE - We don't have the document in storage and none of our peers have the document either

The transitions between these states look like this:

Note that every state can transition to DELETED, either via DocHandle.delete or Repo.delete.

One other point to note is that a DocHandle can be unavailable because we didn't have it in storage and no peers responded to our request for it, but then another peer comes online and sends us sync messages for the document and so it transitions to READY.

You can check what state a handle is in using DocHandle.inState.

Waiting for a handle to be ready​

If all we care about is whether the document is ready then we can use a few different methods.

  • DocHandle.isReady() is a synchronous method which will return true if the document is ready
  • DocHandle.whenReady() is an asynchronous method that will return when the handle is ready
  • DocHandle.doc() is an asynchronous method which will return the value of the document when it is ready
  • DocHandle.docSync() is a synchronous method which returns the value of the document if it is ready. This method will throw if the handle is not ready. Therefore you should guard calls to docSync with calls to isReady

Once the document is ready the value of the document (either DocHandle.doc() or DocHandle.docSync()) will be undefined if the document was unavailable, but otherwise will be an automerge document.

- - + + \ No newline at end of file diff --git a/docs/repositories/ephemeral/index.html b/docs/repositories/ephemeral/index.html index e24d2787..67889261 100644 --- a/docs/repositories/ephemeral/index.html +++ b/docs/repositories/ephemeral/index.html @@ -10,13 +10,13 @@ - - + +

Ephemeral Data

Automerge encourages you to persist most of your application state. Sometimes however there is state which it doesn't make any sense to persist. Good reasons to not persist state are if it changes extremely fast, or is only useful to the user in the context of a live "session" of some kind. One example of such data is cursor positions in collaboratively edited text. We refer to this kind of data as "ephemeral data".

Ephemeral data is associated with a particular document, which means you need to obtain a DocHandle for the document in question in order to send and receive ephemeral data. The rationale for this is that most of the time ephemeral data is related to a particular document. However, if you need to exchange ephemeral data which has no associated document you can always create a blank document and use that.

Sending​

const handle = Repo.find("<some url>")
handle.broadcast({
    "some": "message"
})

The object passed to broadcast will be CBOR encoded so you can send whatever you like.

Receiving​

To receive you listen to the "ephemeral-mesage" event on the DocHandle

const handle = Repo.find("<some url>")
handle.on("ephemeral-message", (message: any) {
    console.log("got an ephemeral message: ", message)
})

The received message will be decoded from CBOR before handing it to the event handler.

- - + + \ No newline at end of file diff --git a/docs/repositories/index.html b/docs/repositories/index.html index 45397e08..5f858315 100644 --- a/docs/repositories/index.html +++ b/docs/repositories/index.html @@ -10,13 +10,13 @@ - - + +

Repositories

@automerge/automerge provides a JSON-like CRDT and a sync protocol, but this still leaves a lot of plumbing to do to use it in an application. @automerge/automerge-repo is that plumbing.

The entry point for an automerge-repo based application is to create a Repo, passing it some form of StorageAdapter - which knows how to save data locally - and zero or more NetworkAdapters, which know how to talk to other peers running automerge-repo.

For example, this snippet creates a Repo which listens for websocket connections and stores data in the local file system: 

import { Repo } from "@automerge/automerge-repo"
import { WebSocketServer } from "ws"
import { NodeWSServerAdapter } from "@automerge/automerge-repo-network-websocket"
import { NodeFSStorageAdapter } from "@automerge/automerge-repo-storage-nodefs"

const socket = new WebSocketServer({ noServer: true })

const repo = new Repo({
    network: [new NodeWSServerAdapter(socket)],
    storage: new NodeFSStorageAdapter(dir),
})

A Repo is a little like a database. It allows you to create and request DocHandles. Once you have a DocHandle you can make changes to it and listen for changes received from other peers.

let doc = repo.create()
// Make a change ourselves and send that to everyone else
doc.change(d => d.text = "hello world")
// Listen for changes from other peers
doc.on("change", {doc} => {
    console.log("new text is ", doc.text)
})

Any changes you make - or which are received from the network - will be stored in the attached storage adapter and distributed to other peers

- - + + \ No newline at end of file diff --git a/docs/repositories/networking/index.html b/docs/repositories/networking/index.html index e5a89248..b0aee45e 100644 --- a/docs/repositories/networking/index.html +++ b/docs/repositories/networking/index.html @@ -10,13 +10,13 @@ - - + +

Networking

There are many ways to talk to other peers. In automerge-repo this is captured by the NetworkAdapter interface. Unlike StorageAdapters a repository can have many (or zero) NetworkAdapters.

"network" is quite a broad term in automerge-repo. It really means "any other instance of Repo which I am communicating with by message passing". This means that as well as network adapters for obvious things like websockets, we also implement network adapters for less traditional channels such as MessageChannel or BroadcastChannel.

Websockets​

The websocket NetworkAdapter has two parts. This is because the websocket protocol requires a server and a client. The parts are named NodeWSServerAdapter and BrowserWebsocketClientAdapter, but don't take these names too seriously, they will both work in a browser or in Node.

Server​

The server side of the adapter is NodeWSServerAdapter, which should be used in combination with the ws library. 

import { WebSocketServer } from "ws"
import { NodeWSServerAdapter } from "@automerge/automerge-repo-network-websocket"

const wss = new WebSocketServer({ port: 8080 });
const adapter = new NodeWSServerAdapter(this.#socket)

Usage with express​

Often you aren't running the websocket server as a standalone thing but instead as part of an existing HTTP server. Here's an example of such a situation in an express app.


import { WebSocketServer } from "ws"
import { NodeWSServerAdapter } from "@automerge/automerge-repo-network-websocket"
import express from "express"

const wss = new WebSocketServer({ noServer: true });
const server = express()
server.on("upgrade", (request, socket, head) => {
    wss.handleUpgrade(request, socket, head, (socket) => {
        wss.emit("connection", socket, request)
    })
})
const adapter = new NodeWSServerAdapter(wss)
server.listen(8080)

Client​

The client side of the connection is BrowserWebsocketClientAdapter. 

import { BrowserWebSocketClientAdapter } from "@automerge/automerge-repo-network-websocket"

const network =  new BrowserWebSocketClientAdapter("ws://localhost:3030")

MessageChannel​

@automerge/automerge-repo-network-messagechannel is a NetworkAdapter for communicating between processes within the same browser using a MessageChannel.

import { MessageChannelNetworkAdapter } from "@automerge/automerge-repo-network-messagechannel"
import { Repo } from "@automerge/automerge-repo"

const { port1: leftToRight, port2: rightToLeft } = new MessageChannel()
const rightToLeft = new MessageChannelNetworkAdapter(rightToLeft)
const leftToRight = new MessageChannelNetworkAdapter(leftToRight)

const left = new Repo({
    network: [leftToRight],
})
const right = new Repo({
    network: [rightToLeft],
})

BroadcastChannel​

@automerge/automerge-repo-network-broadcastchannel is a NetworkAdapter for commuicating between processes in the same browser using a BroadcastChannel. This will in general be quite inefficient because the sync protocol is point-to-point so even though BroadcastChannel is a broadcast channel, we still have to duplicate each message for every peer in the channel. It's better to use MessageChannel if you can, but BroadcastChannel is good in a pinch.

import { BroadcastChannelNetworkAdapter } from "@automerge/automerge-repo-network-broadcastchannel"

const network = new BroadcastChannelNetworkAdapter()
- - + + \ No newline at end of file diff --git a/docs/repositories/storage/index.html b/docs/repositories/storage/index.html index 5f1d3858..f916c898 100644 --- a/docs/repositories/storage/index.html +++ b/docs/repositories/storage/index.html @@ -10,13 +10,13 @@ - - + +

Storage

In automerge-repo "storage" refers to any implementation of StorageAdapter. You can run a Repo without a StorageAdapter but it will be entirely transient and will have to load all it's data from remote peers on every restart.

StorageAdapter is designed to be safe to use concurrently, that is to say it is safe to have multiple Repos talking to the same storage.

There are two built in storage adapters:

IndexedDB​

@automerge/automerge-repo-storage-indexeddb stores data in an IndexedDB in the browser.

import { IndexedDBStorageAdapter } from "@automerge/automerge-repo-storage-indexeddb`

const storage = new IndexedDBStorageAdapter()

You can customize the object database and object store the storage uses, see the docs

As noted above, this is safe for concurrent use so you can have multiple tabs pointing at the same storage. Note that they will not live update (you may want to use a MessageChannel or BroadcastChannel based NetworkAdapter for that) but on refresh the concurrent changes will be merged as per the normal merge rules.

File system​

@automerge/automerge-repo-storage-nodefs is a StorageAdapter which stores its data in a directory on the local filesystem. The location can be customized as per the docs

import { NodeFSStorageAdapter } from "@automerge/automerge-repo-storage-nodefs"
const storage = new NodeFSStorageAdapter()

As with the IndexedDB adapter this adapter is safe for multiple processes to use the same data directory.

Roll your own​

StorageAdapter is designed to be easy to implement. It should be straightforward to build on top of any key/value store which supports range queries.

- - + + \ No newline at end of file diff --git a/docs/the_js_packages/index.html b/docs/the_js_packages/index.html index fcb6a7b2..7e0bb144 100644 --- a/docs/the_js_packages/index.html +++ b/docs/the_js_packages/index.html @@ -10,13 +10,13 @@ - - + +

The JavaScript packages

The javascript API has been through several iterations and is currently split over a few libraries. This page describes how all these pieces fit together.

If you just want to know how to use Automerge in greenfield applications, here's how the library is intended to be used:

Install both the @automerge/automerge and @automerge/automerge-repo packages. Then install the networking and storage plugins you need (typically @automerge/automerge-repo-network-* and @automerge/automerge-repo-storage-*) packages. Take a look at the cookbook for examples of different ways of using these.

When you're making changes to an automerge document you should use the next API

The next API​

Over time we have made a number of changes to the automerge API which are not backwards compatible. In order to not break compatibility we have exposed the new API under a sub module of the library, this submodule used to be called "unstable" to reflect that we were not sure about it - but we're now fairly sure we like it so we're calling it "next" in expectation that it's what the next major version of automerge will look like.

Differences from stable​

  • In the old API javascript strings are represented as scalar strings (see the data model for details) whilst in "next" javascript strings are Text sequences (i.e. they support concurrent insertions and deletions). This means that you should use next.splice to modify strings in the next API. Scalar strings in the next API are represented as instances of the RawString class.
  • The next API exposes the diff and changeAt methods

Using the next API​

There are two ways to use the next API

Subpath Exports​

If you are either using JavaScript in a modern browser or on node > 16 then you can do the following:

import {default as A} from "@automerge/automerge/next"

Note that for this to work in typescript you will need to have the following in your tsconfig.json

    ...
    "module": "NodeNext",
    "moduleResolution": "Node16",
    ...

The { next } module​

If for whatever reason you can't use @automerge/automerge/next then you can do this:

import {next as A} from "@automerge/automerge"

How we got here​

Automerge Classic​

The first version of automerge was implemented in pure javascript and is what we now call automerge Classic. You can find it on NPM and GitHub. This project went through several iterations which changed quite dramatically.

@automerge/automerge-wasm and @automerge/automerge​

More recently we rewrote automerge in Rust and deployed it to javascript by compiling to a wasm package at @automerge/automerge-wasm. This wasm package is something we currently consider to be an implementation detail and should not be depended on by third parties. The @automerge/automerge package offers a very similar API to the original automerge package but implemented by forwarding logic to @automerge/automerge-wasm. It is consequently much faster.

@automerge/automerge-repo​

The core automerge libraries (both the original classic library and the WASM implementation) offer a compact storage format and a network agnostic sync protocol, but they don't actually do the work of wiring these things up to real storage engines (such as filesystems) or transports (such as websockets). automerge-repo implements all of this plumbing and is how we recommend using automerge going forward.

- - + + \ No newline at end of file diff --git a/docs/under-the-hood/merge_rules/index.html b/docs/under-the-hood/merge_rules/index.html index 64aa868e..8f914f1d 100644 --- a/docs/under-the-hood/merge_rules/index.html +++ b/docs/under-the-hood/merge_rules/index.html @@ -10,13 +10,13 @@ - - + +

Merge Rules

info

It isn't important to understand this section to use automerge. You can just let automerge handle merging for you. But it may be interesting to understand.

How does automerge merge concurent changes? Well, let's think about what kinds of concurrent changes are possible. Automerge documents always carry their history with them, so the way to think about two concurrent versions of a document is as the set of changes since some common ancestor.

Here the common ancestor is C and the concurrent changes are (D,E) and (F,G).

Automerge documents are composed of nested maps and lists or simple values or text sequences. We can describe the merge rules by describing the rules for maps, lists, text, and counters independently. In each case we describe how to merge two sets of concurrent changes we refer to as A and B.

Map merge rules​

  • If A sets key xx to a value and B sets key yy to a value and x≠yx \neq y then add both xx and yy to the merged map
  • If A deletes key xx and B makes no change to xx then remove xx from the merged map
  • If A deletes key xx and B sets xx to a new value then set the value of xx to the new value B set in the merged map
  • If both A and B delete key xx then delete xx from the merged map
  • If both A and B set the key xx to some value then randomly choose one value

Note that "randomly choose" means "choose one arbitrarily, but in such a way that all nodes agree on the chosen value".

List merge rules​

To understand the way lists merge you need to know a little about how the operations on lists are expressed. Every element in a list has an ID and operations on the list reference these IDs. When you update an index in a list (using list[<index>] = <value> in a change function in the JS library) the operation which is created references the ID of the element currently at index. Likewise when you delete an element from a list the delete operation which is created references the deleted element at the given index. When you insert elements into a list the insert operation references the ID of the element you are inserting after

In the following then when we say "index xx" that really means "the ID of the element at index xx at the time the operation was created".

  • If A inserts an element after index ii and B inserts an element after index ii then arbitrarily choose one to insert first and then insert the other immediately afterwards
  • If A deletes element at index ii and B updates the element at ii then set the value of ii to the updated value from B
  • If A and B both delete element ii then remove it from the merged list

Note that inserting a run of elements will maintain the insertion order of the replica which generated it. Imagine we have some list [a, b] and say A inserts the sequence [d, e] after a whilst B inserts [f, g] after a. Initially the set of operations are:

Operation IDReference elementValue
ANonea
BAb

The operations after inserting on A are

Operation IDReference elementValue
ANonea
BAb
DBd
EDe

And on B

Operation IDReference elementValue
ANonea
BAb
FBf
GFg

Here you can see that while both F and D insert after the same reference element (B) the following operations reference the element that was just inserted on the local replica. That is, automerge must arbitrarily choose one of either F or D to be inserted after B, but after that the operations stay in the same order as they were inserted on each node. Let's say that A is chosen, then the final order of operations will be

Operation IDReference elementValue
ANonea
BAb
DBd
EDe
FBf
GFg

There are cases where this algorithm does not preserve insertion order - primarily when inserting elements in reverse - but most of the time it does a good job.

Text merge rules​

The characters of a text object are merged using the same logic as lists. For a description of the merge rules for marks see Peritext

Counter merge rules​

Counters are very simple, we just sum all the individual operations from each node.

- - + + \ No newline at end of file diff --git a/docs/under-the-hood/storage/index.html b/docs/under-the-hood/storage/index.html index ac71c90c..8c91dbed 100644 --- a/docs/under-the-hood/storage/index.html +++ b/docs/under-the-hood/storage/index.html @@ -10,13 +10,13 @@ - - + +

Storage

In the getting started section we introduced a simple application which synchronized the value of a counter between any number of tabs. If you close all the tabs and open a new one you will see that the value of the counter is persisted. How is this working? What's going on?

Before we dive into that, try this experiment. Modify the definition of the repo in main.tsx to look like this:

const repo = new Repo({
  network: [], // This part means that we're not sending live changes anywhere
  storage: new IndexedDBStorageAdapter(),
})

Now if you open two tabs with the same URL (including the hash component, the easiest way to achieve this is to open one tab and then duplicate it) you'll notice that the counter value is not updated live between tabs. However, if you increment the count in both tabs and then refresh either tab the count will include the increments from the other tab.

Clearly there is more going on here than just saving the current state of the document somewhere.

Under the hood​

Both tabs initialize a Repo pointing at an IndexedDB storage adapter, because the tabs are on the same domain this means they have access to the same storage.

Let's mess around with this in the browser. First, clear your local IndexedDB for the localhost domain, then open http://localhost:5173 (without a hash component). The browser will update to contain a hash component with the document ID in it. In this example the URL in the browser window is http://localhost:5173/#automerge:3RFyJzsLsZ7MsbG98rcuZ4FqtGW7, so the document URL is automerge:3RFyJzsLsZ7MsbG98rcuZ4FqtGW7.

Open the browser tools and take a look at IndexedDB you'll see a database called automerge and within that an object store called automerge. For me, in Firefox, this looks like:

IndexedDB browser tools

You can see that there is a key which looks roughly like our document URL (it doesn't have the automerge: prefix) and some kind of value. If we expand that we see:

IndexedDB detailed

If you're not familiar with IndexedDB this might be a little confusing. IndexedDB is a sort of key/value store where the keys are arrays. So what we are seeing here is a binary array (the binary: Object part in the above screenshot) stored under the key ["3RFyJzsLsZ7MsbG98rcuZ4FqtGW7", "incremental", "0290cdc2dcebc1ecb3115c3635bf1cb0f857ce971d9aab1c44a0d3ab19a88cd8"].

Okay, so creating a document (which is what happens when we load the page) stores a binary array under some key in the object database. This binary array is a single "incremental" change. An incremental change is not the entire history of the document but just some set of chagnes to the document. In this case it's the change that initializes the document with a "counter" field.

Now click the "count" button and take another look at the IndexedDB.

IndexedDB snapshot

Well, there's still one entry, but it's changed. The [.., "incremental", ..] key has been deleted and replaced with [.., "snapshot", ..]. What's happened here? Every time you make a change automerge-repo saves that change to your storage adapters. Occasionally automerge-repo will decide that it's time to "compact" the document, it will take every change that has been written to storage so far (in this case, every key beginning with [<document URL>, .., ..] and combine them into a single snapshot and then save it as this [.., "snapshot", ..] key.

All well and good in one tab. Open a new tab with the same URL (including the hash) and click the count button a few times in both tabs. If you look at the IndexedDB browser tools (in either tab, it's shared between them) you'll something like this:

IndexedDB many keys

You can see here that there are two snapshot files. This is because when each tab compacts incremental changes and then deletes the original incremental files, it only deletes the incremental changes it had previously loaded. This is what makes it safe to use concurrently, because it only deletes data which is incorporated into the compacted document. But the real magic comes with how this is loaded. If you load another tab with the same URL it will sum the counts from both the previous tabs. This works because when the repo starts up it loads all the changes it can find in storage and merges them which it can do because automerge is a CRDT.

The storage model​

The objective of the storage engine in automerge-repo is to be easy to implement over a wide range of backing stores (e.g. an S3 bucket, or a postgres database, or a local directory) and support compaction without requiring any concurrency control on the part of the implementor. Compaction is crucial to make the approach of storing every change that is made to a document feasible.

The simplest model of storage is a key/value model. We could attempt to build storage on top of such a model by using the document ID as a key, appending new changes for a document to that key and occasionally compacting the document and rewriting the value at that key entirely. The problem with this is that it makes it complicated to use the storage engine from multiple processes. Imagine multiple processes are making changes to a document and writing them to the storage backend. If both of these processes decide to compact at the same time then the storage engine would need to have some kind of transaction to ensure that between the time a compacting process read from storage and then wrote to it no other process added new changes to storage. This is not hard for something like a postgres database, but it's very fiddly for simple mediums like a directory on the local filesystem.

What we want to be able to do then is to know that if we are writing a compacted document to storage we will never overwrite data which contains changes we did not compact. Conveniently the set of changes in the document is uniquely identified by the heads of the document. This means that if we use the tuple (document ID, <heads of document>) as the key to the storage we know that even if we overwrite data another process has written it must contain the same changes as the data we are writing.

Of course, we also want to remove the un-compacted data. A compacting process can't just delete everything because another process might have written new changes since it started compaction. Each process then needs to keep track of every change it has loaded from storage and then when compacting only delete those changes.

The upshot of all this then is that our model for storage is not a key value store with document IDs as keys and byte arrays as values, but instead a slightly more complex model where the keys are arrays of the form [<document ID>, <chunk type>, <chunk identifier>] where chunk type is either "snapshot" or "incremental" and the chunk ID is either the heads of the documnt at compaction time or the hash of the change bytes respectively. The storage backend then must implement range queries so the storage system can do things like "load all the chunks for document ID x".

In typescript that looks like this:

export type  StorageKey = string[]

export abstract class StorageAdapter {
  abstract load(key: StorageKey): Promise<Uint8Array | undefined>
  abstract save(key: StorageKey, data: Uint8Array): Promise<void>
  abstract remove(key: StorageKey): Promise<void>
  abstract loadRange(keyPrefix: StorageKey): Promise<{key: StorageKey, data: Uint8Array}[]>
  abstract removeRange(keyPrefix: StorageKey): Promise<void>
}
- - + + \ No newline at end of file diff --git a/index.html b/index.html index f60b6a22..c1e0b45b 100644 --- a/index.html +++ b/index.html @@ -10,13 +10,13 @@ - - + +

Build local-first software

Automerge is a library of data structures for building collaborative applications.

Get Started
docu_tree

Automatic merging

Automerge is a Conflict-Free Replicated Data Type (CRDT), which allows concurrent changes on different devices to be merged automatically without requiring any central server.

Network-agnostic

Use any connection-oriented network protocol: client-server, peer-to-peer, or local. Or use unidirectional messaging: send an Automerge file as an email attachment or store it on a file server.

Portable

Implemented in JavaScript and Rust, with FFI bindings across platforms including iOS, Electron, Chrome, Safari, Edge, Firefox, and more.

- - + + \ No newline at end of file diff --git a/markdown-page/index.html b/markdown-page/index.html index 1c984feb..97200b4c 100644 --- a/markdown-page/index.html +++ b/markdown-page/index.html @@ -10,13 +10,13 @@ - - + +

Markdown page example

You don't need React to write simple standalone pages.

- - + + \ No newline at end of file diff --git a/sitemap.xml b/sitemap.xml index ba59c574..cc3a6903 100644 --- a/sitemap.xml +++ b/sitemap.xml @@ -1 +1 @@ -https://automerge.github.io/blog/weekly0.5https://automerge.github.io/blog/archive/weekly0.5https://automerge.github.io/blog/automerge-2/weekly0.5https://automerge.github.io/blog/tags/weekly0.5https://automerge.github.io/blog/tags/hello/weekly0.5https://automerge.github.io/blog/welcome/weekly0.5https://automerge.github.io/markdown-page/weekly0.5https://automerge.github.io/docs/api/weekly0.5https://automerge.github.io/docs/concepts/weekly0.5https://automerge.github.io/docs/cookbook/modeling-data/weekly0.5https://automerge.github.io/docs/documents/weekly0.5https://automerge.github.io/docs/documents/conflicts/weekly0.5https://automerge.github.io/docs/documents/counters/weekly0.5https://automerge.github.io/docs/documents/lists/weekly0.5https://automerge.github.io/docs/documents/text/weekly0.5https://automerge.github.io/docs/documents/values/weekly0.5https://automerge.github.io/docs/glossary/weekly0.5https://automerge.github.io/docs/hello/weekly0.5https://automerge.github.io/docs/quickstart/weekly0.5https://automerge.github.io/docs/repositories/weekly0.5https://automerge.github.io/docs/repositories/dochandles/weekly0.5https://automerge.github.io/docs/repositories/ephemeral/weekly0.5https://automerge.github.io/docs/repositories/networking/weekly0.5https://automerge.github.io/docs/repositories/storage/weekly0.5https://automerge.github.io/docs/the_js_packages/weekly0.5https://automerge.github.io/docs/under-the-hood/merge_rules/weekly0.5https://automerge.github.io/docs/under-the-hood/storage/weekly0.5https://automerge.github.io/weekly0.5 \ No newline at end of file +https://automerge.github.io/blog/weekly0.5https://automerge.github.io/blog/2023/11/06/automerge-repo/weekly0.5https://automerge.github.io/blog/archive/weekly0.5https://automerge.github.io/blog/automerge-2/weekly0.5https://automerge.github.io/blog/tags/weekly0.5https://automerge.github.io/blog/tags/hello/weekly0.5https://automerge.github.io/blog/welcome/weekly0.5https://automerge.github.io/markdown-page/weekly0.5https://automerge.github.io/docs/api/weekly0.5https://automerge.github.io/docs/concepts/weekly0.5https://automerge.github.io/docs/cookbook/modeling-data/weekly0.5https://automerge.github.io/docs/documents/weekly0.5https://automerge.github.io/docs/documents/conflicts/weekly0.5https://automerge.github.io/docs/documents/counters/weekly0.5https://automerge.github.io/docs/documents/lists/weekly0.5https://automerge.github.io/docs/documents/text/weekly0.5https://automerge.github.io/docs/documents/values/weekly0.5https://automerge.github.io/docs/glossary/weekly0.5https://automerge.github.io/docs/hello/weekly0.5https://automerge.github.io/docs/quickstart/weekly0.5https://automerge.github.io/docs/repositories/weekly0.5https://automerge.github.io/docs/repositories/dochandles/weekly0.5https://automerge.github.io/docs/repositories/ephemeral/weekly0.5https://automerge.github.io/docs/repositories/networking/weekly0.5https://automerge.github.io/docs/repositories/storage/weekly0.5https://automerge.github.io/docs/the_js_packages/weekly0.5https://automerge.github.io/docs/under-the-hood/merge_rules/weekly0.5https://automerge.github.io/docs/under-the-hood/storage/weekly0.5https://automerge.github.io/weekly0.5 \ No newline at end of file