The Claim-Check Pattern for Oversized Spatial Payloads

Write the body to object storage under a key derived from its own SHA-256, publish the envelope only after that write returns, verify the digest after every read, and give the bucket a longer lifecycle than the topic’s retention — the ordering is what stops a fast consumer resolving a key that does not exist yet.

This guide sits under Streaming & Chunking Large Geometry Payloads, within Spatial Payload Routing & Parsing. That topic compares the three strategies; this one is the implementation detail of the recommended default.

When to use this pattern

  • Geometry size has a long upper tail, so a per-message limit is met occasionally rather than constantly.
  • Consumers are geographically sharded and discard most events, which is where lazy resolution pays.
  • The event must stay atomic — one message, one event — which chunking cannot provide.

The three ordering rules

The window between publish and store Two producer orderings are traced against a consumer polling at the head of the topic. In the first, the envelope is published and the body is written afterwards. A consumer polling every fifty milliseconds sees the envelope, resolves the reference, and receives a not-found error because the object write has not completed — a window of perhaps two hundred milliseconds for a large body, widening under load precisely when the pipeline can least absorb it. The error is indistinguishable from an object that was deleted, so the consumer cannot tell whether to retry or dead-letter, and retrying burns budget on a race that resolves itself. In the second, the body is written first and the envelope published only after that write returns. There is no window: any envelope a consumer can see refers to an object that already exists. The cost of the correct order is that a producer crashing between the two leaves an object with no event referencing it, which is a garbage-collection problem — a periodic sweep of unreferenced keys older than the topic's retention — rather than a correctness one. publish, then store — a race with every consumer publish envelope object write completes the window consumer resolves → NoSuchKey, indistinguishable from a deletion so it cannot tell whether to retry or dead-letter, and retrying burns budget on a race the window widens with body size and under load — when the pipeline can least absorb it store, then publish — no window exists object write completes publish envelope any envelope a consumer can see refers to an object that already exists The cost of the correct order: a producer crashing between the two leaves an object nothing references. That is a garbage-collection problem — sweep unreferenced keys older than the topic's retention — rather than a correctness one.
Figure 1. Both orders have a failure mode. One produces an orphaned object you can sweep on a schedule; the other produces an error consumers cannot classify.

Complete runnable implementation

python 
import hashlib
import json
from dataclasses import asdict, dataclass

from shapely.geometry import shape

INLINE_LIMIT = 256 * 1024


class BodyMismatch(Exception):
    """The stored body is not the one the envelope described."""


@dataclass(slots=True)
class Envelope:
    feature_id: str
    occurred_at: str
    crs: str                                   # always explicit
    bbox: tuple[float, float, float, float]    # stays here — see gotcha 1
    vertex_count: int
    geometry: dict | None = None
    body_key: str | None = None
    body_sha256: str | None = None
    body_bytes: int | None = None


async def publish(geometry: dict, feature_id: str, occurred_at: str,
                  store, broker) -> Envelope:
    geom = shape(geometry)
    body = json.dumps(geometry, separators=(",", ":")).encode()

    env = Envelope(
        feature_id=feature_id, occurred_at=occurred_at, crs="EPSG:4326",
        bbox=geom.bounds, vertex_count=_count_vertices(geom),
    )

    if len(body) < INLINE_LIMIT:
        env.geometry = geometry
    else:
        digest = hashlib.sha256(body).hexdigest()
        # Content-addressed: a retry writes identical bytes to the same key,
        # so the write is idempotent and needs no coordination.
        env.body_key = f"geom/{digest[:2]}/{digest}.json"
        env.body_sha256, env.body_bytes = digest, len(body)
        await store.put(env.body_key, body)     # FIRST

    await broker.send(json.dumps(asdict(env)).encode())   # SECOND
    return env


async def resolve(env: Envelope, store, area_of_interest=None) -> dict | None:
    """Fetch the body only if this event is relevant, then verify it."""
    # Lazy resolution: most of the saving is the events never fetched at all.
    if area_of_interest is not None and not _intersects(env.bbox, area_of_interest):
        return None

    if env.geometry is not None:
        return env.geometry

    body = await store.get(env.body_key)

    # A truncated read of a FeatureCollection frequently parses: it ends
    # after a complete feature and yields a valid, smaller collection. The
    # length and digest checks are the only things that catch it.
    if len(body) != env.body_bytes:
        raise BodyMismatch(f"{env.body_key}: {len(body)} != {env.body_bytes}")
    if hashlib.sha256(body).hexdigest() != env.body_sha256:
        raise BodyMismatch(f"{env.body_key}: digest mismatch")

    return json.loads(body)


async def sweep_orphans(store, broker_retention_seconds: int, referenced) -> int:
    """Delete bodies no event references, once replay can no longer need them.

    Sweeping earlier than the topic's retention deletes objects that a replay
    would still resolve, which converts a harmless orphan into a broken event.
    """
    removed = 0
    async for key, age_seconds in store.list_with_age("geom/"):
        if age_seconds > broker_retention_seconds and key not in referenced:
            await store.delete(key)
            removed += 1
    return removed
Retention has to outlast the topic, not match it A topic retains seven days of events. The bucket holding referenced bodies is configured with a three-day lifecycle, which looks generous next to typical object storage costs and is set by someone who has never replayed the topic. For the first three days everything works: every envelope resolves and every body is present. On day four the objects for days one to three have expired while their envelopes are still in the log, so a replay from the beginning of retention resolves references to objects that no longer exist. The failure appears only during a replay, which means only during an incident, which is the moment the pipeline is least able to absorb an additional unknown. Setting the bucket lifecycle beyond the topic's retention — and beyond the age of any dead-letter archive that can be replayed — costs storage and removes the failure entirely. The orphan sweep then handles the other direction, deleting bodies whose events are gone, and it must use the same boundary. topic retention 7 days · bucket lifecycle 3 days envelopes in the log day 1 ────────────────────────────────────────────── day 7 bodies in the bucket days 5–7 only days 1–4 expired — envelopes still present, bodies gone The failure appears only during a replay …which means only during an incident, which is the moment the pipeline is least able to absorb an additional unknown. Set the lifecycle beyond topic retention AND beyond any replayable dead-letter archive The orphan sweep uses the same boundary from the other direction: delete bodies whose events are gone, never sooner.
Figure 2. The bucket lifecycle and the topic retention are two halves of one number. Configuring them in different systems is why they drift apart.

Parameter reference

Name Type Spatial constraint Default
INLINE_LIMIT int Chosen so the 99th percentile of measured sizes stays inline 262144
body_key str sha256(body) — content-addressed, so retries are idempotent —
body_sha256 str Verified after every read; a truncated collection still parses —
bbox tuple Must stay in the envelope, or lazy resolution is impossible —
Bucket lifecycle days > topic retention and any replayable archive 14
Orphan sweep age seconds Same boundary, from the other direction retention

Gotchas and spatial edge cases

  1. Moving the bounding box into the body destroys the pattern’s main benefit. Lazy resolution is why a geographically sharded consumer costs almost nothing: it discards most events without a storage round trip. With the bbox in the body, every consumer fetches every object to learn the event was irrelevant, and the claim check becomes strictly worse than inlining.

  2. A truncated body can parse successfully. A FeatureCollection cut after a complete feature is a valid collection with fewer features, so json.loads succeeds and a coverage calculation quietly reports a smaller area. The length check catches it; the digest check catches the case where the length is right and the content is not.

  3. The key must not encode anything mutable. Putting a date or a tenant name in the key path means two producers with different clocks or different configuration write the same body twice, and the idempotence the content address provides is lost. The digest prefix used for path sharding is fine because it derives from the body.

  4. Presigned URLs expire, and replay is exactly when they have. If envelopes carry presigned fetch URLs rather than keys, a replay a week later resolves URLs that are no longer valid. Carry the key and let the consumer sign at fetch time.

  5. The store’s consistency model matters. Most object stores are now strongly consistent for new writes, but a read-after-write against a stale replica returns not-found even with correct ordering. Treat a not-found on a freshly published envelope as retryable and a not-found on an old one as terminal, and use the envelope’s timestamp to tell them apart.

  6. The digest cannot double as the idempotency key. Two genuinely different events can carry an identical geometry — the same boundary re-published by two sources — and they hash the same. Event identity comes from Event Key Generation for Spatial Data; the digest identifies bytes.

Content addressing makes a retry free and a duplicate harmless The same boundary geometry is published twice: once by a producer that timed out and retried, and once by a second service that independently republished the same shape. Under a content-addressed key both writes target the same object, so the second write replaces identical bytes and costs nothing to reconcile — there is exactly one object for one set of bytes, however many events reference it, and the orphan sweep has one thing to consider. Under an event-id key the same bytes are written twice under two names, so storage holds two copies, the sweep must decide about each independently, and a later deduplication of the events leaves one object referenced and one not, with no way to tell from the object itself. The content address is also what makes the producer retry safe without coordination: no lock, no check-then-write, and no window in which a partially written object is visible under a name something already references. the same bytes published twice — a retry, and an independent republish key = sha256(body) geom/4f/4f2c…json ← both writes one object per set of bytes, however many events reference it the retry needs no lock, no check-then-write, no coordination the orphan sweep has one thing to consider key = event id geom/evt-8871.json geom/evt-8872.json two copies of identical bytes, two names the sweep must decide about each independently after event dedup, one is referenced and one is not …and nothing in the object itself says which. The content address removes the question rather than answering it.
Figure 3. The digest is doing two jobs: it verifies the body on read, and it makes the write idempotent on the way in. Only the first is obvious from the code.

Verification

python 
import json
import pytest
from shapely.geometry import MultiPolygon, Polygon, mapping


def big() -> dict:
    return mapping(MultiPolygon([
        Polygon([(i, j), (i, j + 0.9), (i + 0.9, j + 0.9), (i + 0.9, j)])
        for i in range(120) for j in range(120)
    ]))


@pytest.mark.asyncio
async def test_store_happens_before_publish(store, broker):
    """The ordering rule, asserted against the recorded call sequence."""
    await publish(big(), "f-1", "2026-08-08T10:00:00Z", store, broker)
    assert store.calls[0][0] == "put"
    assert store.calls[0][1] < broker.calls[0][1]      # timestamps


@pytest.mark.asyncio
async def test_envelope_stays_small_and_routable(store, broker):
    env = await publish(big(), "f-2", "2026-08-08T10:00:00Z", store, broker)
    assert len(broker.sent[-1]) < 2048
    assert env.bbox is not None and env.vertex_count > 0


@pytest.mark.asyncio
async def test_irrelevant_event_is_never_fetched(store, broker):
    """Where the saving actually comes from."""
    env = await publish(big(), "f-3", "2026-08-08T10:00:00Z", store, broker)
    before = store.reads
    assert await resolve(env, store, area_of_interest=(200, 200, 210, 210)) is None
    assert store.reads == before


@pytest.mark.asyncio
async def test_truncated_body_is_rejected(store, broker):
    env = await publish(big(), "f-4", "2026-08-08T10:00:00Z", store, broker)
    store.truncate(env.body_key, keep=env.body_bytes // 2)
    with pytest.raises(BodyMismatch):
        await resolve(env, store)


@pytest.mark.asyncio
async def test_retry_writes_the_same_key(store, broker):
    """Content addressing makes the producer retry harmless."""
    a = await publish(big(), "f-5", "2026-08-08T10:00:00Z", store, broker)
    b = await publish(big(), "f-5", "2026-08-08T10:00:00Z", store, broker)
    assert a.body_key == b.body_key

The third test is the one that keeps the pattern honest. It fails the moment someone moves the bounding box into the body for tidiness, and that change would otherwise show up only as a slow rise in storage read costs that nobody attributes to a schema edit.