10,000 writes of 500 bytes second

If somebody writes data, it gets piped straight to a cold cache stream.

After 15 seconds or 100MB we batch the cold cache stream.

Then spawn a process to convert the last cold cache stream into a cold cache state. Ack level 1.

The cold cache state is an O(1) radix format, where 1 is the length of the key, that can return an individual value. Ack level 2.

The cold cache also is serializable into a radix file format that can be stream replicated.

Based on the updates, the radix is chunked into 100MB, where any 100MB section that is different than before gets stream written to cold storage at the same or different interval. Ack level 3.

If somebody reads data, in a new process we first reverse scan the current stream batch to warm cache it if it exists.

We then read the radix and merge it with the warm cache response and reply immediately if data exists.

Finally we check to see if we've read from cold storage and if we have not (or if we haven't in a while, like 15 seconds multiplied by 100) we read stream from cold storage and (optionally immediately reply on matches) and then run the same steps to convert this into the cold cache state.

For cost optimizations, you might want to provide the developer with a deterministic key generator helper that mashes primary key (immutable) indices together into a radix hierarchy.

We theoretically should be able to delete the LRU 100MB radix chunks if we run out of space.

-------------------------------------- STREAM >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>

%'1024'#'"marknadal/says/1234567890/asdf"'.'"who"'='#"marknadal"'.'"when"'='1234567890'.'"what"'='"Hello world!"'>'1234567890'#'"marknadal"'.'"last"'='#"marknadal/says/1234567890/asdf"'.'"says"'='#"marknadal/says"'>'1234567890'#'"marknadal/says"'.'"marknadal/says/1234567890/asdf"'='#"marknadal/says/1234567890/asdf"'>'1234567890'

%'1024'#'"ambernadal/says/1234567890/asdf"'.'"who"'='#"ambernadal"'.'"when"'='1234567890'.'"what"'='"Hello world!"'>'1234567890'#'"ambernadal"'.'"last"'='#"ambernadal/says/1234567890/asdf"'.'"says"'='#"ambernadal/says"'>'1234567890'#'"ambernadal/says"'.'"ambernadal/says/1234567890/asdf"'='#"ambernadal/says/1234567890/asdf"'>'1234567890'

%'1024'#'"jessegibson/says/1234567890/asdf"'.'"who"'='#"jessegibson"'.'"when"'='1234567890'.'"what"'='"Hello world!"'>'1234567890'#'"jessegibson"'.'"last"'='#"jessegibson/says/1234567890/asdf"'.'"says"'='#"jessegibson/says"'>'1234567890'#'"jessegibson/says"'.'"jessegibson/says/1234567890/asdf"'='#"jessegibson/says/1234567890/asdf"'>'1234567890'

%'1024'#'"marknadal/says/1234567891/fdsa"'.'"who"'='#"marknadal"'.'"when"'='1234567891'.'"what"'='"Hello universe!"'>'1234567891'#'"marknadal"'.'"last"'='#"marknadal/says/1234567891/fdsa"'.'"says"'='#"marknadal/says"'>'1234567891'#'"marknadal/says"'.'"marknadal/says/1234567891/fdsa"'='#"marknadal/says/1234567891/fdsa"'>'1234567891'

--------------------------------------- STATE >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>

=@
	'#'
		'ambernadal'
			=@
				'.'
					'last' = '#"ambernadal/says/1234567890/asdf"'>'1234567890'
					'says' = '#"ambernadal/says"'>'1234567890'
			'/says'
				=@
					'.ambernadal/says/1234567890/asdf' = '#"ambernadal/says/1234567890/asdf"'>'1234567890'
				'/1234567890/asdf'
					=@
						'.wh'
							'at' = '"Hello world!"'>'1234567890'
							'en' = '1234567890'>'1234567890'
							'o' = '#"ambernadal"'>'1234567890'
		'jessegibson'
			=@
				'.'
					'last' = '#"jessegibson/says/1234567890/asdf"'>'1234567890'
					'says' = '#"jessegibson/says"'>'1234567890'
			'/says'
				=@
					'.jessegibson/says/1234567890/asdf' = '#"jessegibson/says/1234567890/asdf"'>'1234567890'
				'/1234567890/asdf'
					=@
						'.wh'
							'at' = '"Hello world!"'>'1234567890'
							'en' = '1234567890'>'1234567890'
							'o' = '#"jessegibson"'>'1234567890'
		'marknadal'
			=@
				'.'
					'last' = '#"marknadal/says/1234567891/fdsa"'>'1234567891'
					'says' = '#"marknadal/says"'>'1234567891'
			'/says'
				=@
					'.marknadal/says/123456789'
						'0/asdf' = '#"marknadal/says/1234567890/asdf"'>'1234567890'
						'1/fads' = '#"marknadal/says/1234567891/fads"'>'1234567891'
				'/123456789'
					'0/asdf'
						=@
							'.wh'
								'at' = '"Hello world!"'>'1234567890'
								'en' = '1234567890'>'1234567890'
								'o' = '#"marknadal"'>'1234567890'
					'1/fads'
						=@
							'.wh'
								'at' = '"Hello universe!"'>'1234567891'
								'en' = '1234567891'>'1234567891'
								'o' = '#"marknadal"'>'1234567891'