Embedded Database Part 1: Append-Only Commit Log
High-throughput database engines (LSM-trees, Bitcask) write updates sequentially to an append-only commit log. Sequential streaming eliminates random I/O seek latency, saturating disk write bandwidth 📜.
import io
import struct
class CommitLog:
def __init__(self):
self.stream = io.BytesIO()
def append(self, key, val):
offset = self.stream.tell()
kb, vb = key.encode(), val.encode()
self.stream.write(struct.pack("!II", len(kb), len(vb)) + kb + vb)
return offset
def read_at(self, offset):
self.stream.seek(offset)
klen, vlen = struct.unpack("!II", self.stream.read(8))
return self.stream.read(klen).decode(), self.stream.read(vlen).decode()
log = CommitLog()
off = log.append("user:101", "active")
print(f"Read back from log: {log.read_at(off)} 📜")
# 110. Embedded Database Part 1: Append-Only Commit Log High-throughput database engines (LSM-trees, Bitcask) write updates sequentially to an append-only commit log. Sequential streaming eliminates random I/O seek latency, saturating disk write bandwidth 📜. ```python import io import struct class CommitLog: def __init__(self): self.stream = io.BytesIO() def append(self, key, val): offset = self.stream.tell() kb, vb = key.encode(), val.encode() self.stream.write(struct.pack("!II", len(kb), len(vb)) + kb + vb) return offset def read_at(self, offset): self.stream.seek(offset) klen, vlen = struct.unpack("!II", self.stream.read(8)) return self.stream.read(klen).decode(), self.stream.read(vlen).decode() log = CommitLog() off = log.append("user:101", "active") print(f"Read back from log: {log.read_at(off)} 📜") ``` Because append-only files grow indefinitely, production engines run background compaction routines to merge old segments and discard overwritten tombstoned keys ⚡! --- ### Question Why do modern write-heavy storage engines favor append-only logs over in-place record updates? 🧠- [ ] a. Sequential disk writes achieve maximum I/O throughput by avoiding random head seeks and write amplification 📜 - [ ] b. Append-only files are mathematically guaranteed to never exceed 1 GB in size - [ ] c. Linux operating systems do not support seek operations on disk files - [ ] d. In-place file modifications corrupt operating system file descriptors --- ### Challenge Build `CommitLog` using `io.BytesIO()`. Append `('k1', 'v1')`, then `('k2', 'v2')`. Read from offset 0 and print the `(key, value)` tuple. 🎯 <details> <summary>Solution</summary> ```python import io import struct class CommitLog: def __init__(self): self.stream = io.BytesIO() def append(self, key, val): offset = self.stream.tell() kb, vb = key.encode(), val.encode() self.stream.write(struct.pack('!II', len(kb), len(vb)) + kb + vb) return offset def read_at(self, offset): self.stream.seek(offset) klen, vlen = struct.unpack('!II', self.stream.read(8)) return self.stream.read(klen).decode(), self.stream.read(vlen).decode() log = CommitLog() log.append('k1', 'v1') log.append('k2', 'v2') print(log.read_at(0)) ``` </details>