diff --git a/examples/topologies/multi-region/cli/Dockerfile b/examples/topologies/multi-region/cli/Dockerfile new file mode 100644 index 00000000..99f733cb --- /dev/null +++ b/examples/topologies/multi-region/cli/Dockerfile @@ -0,0 +1,23 @@ +FROM golang:1.19-alpine3.17 AS build + +RUN apk update && apk add git + +RUN go install github.com/nats-io/nats-server/v2@v2.9.15 +RUN go install github.com/nats-io/natscli/nats@main +RUN go install github.com/nats-io/nsc/v2@v2.7.8 + +FROM alpine:3.17 + +RUN apk add bash curl + +COPY --from=build /go/bin/nats-server /usr/local/bin/ +COPY --from=build /go/bin/nats /usr/local/bin/ +COPY --from=build /go/bin/nsc /usr/local/bin/ + +WORKDIR /app + +COPY . . + +ENTRYPOINT ["bash"] + +CMD ["main.sh"] diff --git a/examples/topologies/multi-region/cli/GLOBAL.json b/examples/topologies/multi-region/cli/GLOBAL.json new file mode 100644 index 00000000..773321fa --- /dev/null +++ b/examples/topologies/multi-region/cli/GLOBAL.json @@ -0,0 +1,23 @@ +{ + "name": "GLOBAL", + "subjects": [ + "js.in.global.>" + ], + "retention": "limits", + "max_consumers": -1, + "max_msgs_per_subject": -1, + "max_msgs": -1, + "max_bytes": -1, + "max_age": 0, + "max_msg_size": -1, + "storage": "file", + "discard": "old", + "num_replicas": 3, + "duplicate_window": 120000000000, + "sealed": false, + "deny_delete": false, + "deny_purge": false, + "allow_rollup_hdrs": false, + "allow_direct": false, + "mirror_direct": false +} diff --git a/examples/topologies/multi-region/cli/ORDERS_CENTRAL.json b/examples/topologies/multi-region/cli/ORDERS_CENTRAL.json new file mode 100644 index 00000000..b2cd4c20 --- /dev/null +++ b/examples/topologies/multi-region/cli/ORDERS_CENTRAL.json @@ -0,0 +1,39 @@ +{ + "name": "ORDERS_CENTRAL", + "subjects": [ + "js.in.orders_central" + ], + "retention": "limits", + "max_consumers": -1, + "max_msgs_per_subject": -1, + "max_msgs": -1, + "max_bytes": -1, + "max_age": 0, + "max_msg_size": -1, + "storage": "file", + "discard": "old", + "num_replicas": 3, + "duplicate_window": 120000000000, + "placement": { + "cluster": "", + "tags": [ + "region:central" + ] + }, + "sources": [ + { + "name": "ORDERS_EAST", + "filter_subject": "js.in.orders_east" + }, + { + "name": "ORDERS_WEST", + "filter_subject": "js.in.orders_west" + } + ], + "sealed": false, + "deny_delete": false, + "deny_purge": false, + "allow_rollup_hdrs": false, + "allow_direct": false, + "mirror_direct": false +} diff --git a/examples/topologies/multi-region/cli/ORDERS_EAST.json b/examples/topologies/multi-region/cli/ORDERS_EAST.json new file mode 100644 index 00000000..33584ee8 --- /dev/null +++ b/examples/topologies/multi-region/cli/ORDERS_EAST.json @@ -0,0 +1,39 @@ +{ + "name": "ORDERS_EAST", + "subjects": [ + "js.in.orders_east" + ], + "retention": "limits", + "max_consumers": -1, + "max_msgs_per_subject": -1, + "max_msgs": -1, + "max_bytes": -1, + "max_age": 0, + "max_msg_size": -1, + "storage": "file", + "discard": "old", + "num_replicas": 3, + "duplicate_window": 120000000000, + "placement": { + "cluster": "", + "tags": [ + "region:east" + ] + }, + "sources": [ + { + "name": "ORDERS_WEST", + "filter_subject": "js.in.orders_west" + }, + { + "name": "ORDERS_CENTRAL", + "filter_subject": "js.in.orders_central" + } + ], + "sealed": false, + "deny_delete": false, + "deny_purge": false, + "allow_rollup_hdrs": false, + "allow_direct": false, + "mirror_direct": false +} diff --git a/examples/topologies/multi-region/cli/ORDERS_WEST.json b/examples/topologies/multi-region/cli/ORDERS_WEST.json new file mode 100644 index 00000000..db6c0c29 --- /dev/null +++ b/examples/topologies/multi-region/cli/ORDERS_WEST.json @@ -0,0 +1,39 @@ +{ + "name": "ORDERS_WEST", + "subjects": [ + "js.in.orders_west" + ], + "retention": "limits", + "max_consumers": -1, + "max_msgs_per_subject": -1, + "max_msgs": -1, + "max_bytes": -1, + "max_age": 0, + "max_msg_size": -1, + "storage": "file", + "discard": "old", + "num_replicas": 3, + "duplicate_window": 120000000000, + "placement": { + "cluster": "", + "tags": [ + "region:west" + ] + }, + "sources": [ + { + "name": "ORDERS_EAST", + "filter_subject": "js.in.orders_east" + }, + { + "name": "ORDERS_CENTRAL", + "filter_subject": "js.in.orders_central" + } + ], + "sealed": false, + "deny_delete": false, + "deny_purge": false, + "allow_rollup_hdrs": false, + "allow_direct": false, + "mirror_direct": false +} diff --git a/examples/topologies/multi-region/cli/README.md b/examples/topologies/multi-region/cli/README.md new file mode 100644 index 00000000..c0d209d6 --- /dev/null +++ b/examples/topologies/multi-region/cli/README.md @@ -0,0 +1,107 @@ +# Multi Region NATS Cluster + +This demonstrates a very basic NATS Cluster using only routes across 3 regions. + +This should be used in cases where a multi region setup is needed and one requires a stream to be globally deployed and have globally consistent message deduplication. + +## Constraints + + * This should be deployed in networks with generally below 100ms latency. For example in GCP using Tier-1 network connectivity in US east, west and central. + * Streams should generally be R3 when stretched out of a single region to mitigate the big exposure they would have to latency if R5 + * Multi-region streams should not be the default, ideally these are only used for those cases where global deduplication is needed + * For general streams where eventual consistency is acceptible streams should be bound to a region and replicated into others if desired + +## Configuration guidelines + +### Server Tags + +Each server is tagged with a regional tag, for example, `region:east` or `region:west`. The servers are configured with anti-affinity on this tag using the following configuration: + +``` +jetstream { + unique_tag: "region:" +} +``` + +This ensures that streams that are not specifically bound to a single region will be split across the 3 regions. + +### Configuring single region streams + +``` +$ nats stream add --tag region:east --replicas 3 EAST +... +Cluster Information: + + Name: c1 + Leader: n1-east + Replica: n2-east, current, seen 1ms ago + Replica: n3-east, current, seen 0s ago +``` + +Note the servers are `n1-east`, `n2-east`, `n3-east`. + +### Configuring a global stream + +``` +$ nats stream add --replicas 3 GLOBAL +... +Cluster Information: + + Name: c1 + Leader: n1-central + Replica: n2-east, current, seen 0s ago + Replica: n3-west, current, seen 0s ago + +``` + +Note here we give no placement directive so the server will spread it across regions due to the `unique_tag` setting. Servers are `n1-central`, `n2-east` and `n3-west` - 1 per region. + +### Configuring replicated streams + +To facilitate an eventually consistent but single config set up we show how to create 3 streams: + + * `ORDERS_EAST` listening on subjects `js.in.orders_east` + * `ORDERS_WEST` listening on subjects `js.in.orders_west` + * `ORDERS_CENTRAL` listening on subjects `js.in.orders.central` + +We then use server mappings in each region to map `js.in.orders` to the in-region subject: + +``` +accounts { + one: { + jetstream: enabled + mappings: { + js.in.orders: js.in.orders_central + } + } +} +``` + +We can now publish to one subject and depend on which region we connect to the local stream will handle - then replicate - the data: + +``` +$ nats --context contexts/central-user.json req js.in.orders 1 +{"stream":"ORDERS_CENTRAL", "seq":4} + +$ nats --context contexts/east-user.json req js.in.orders 1 +{"stream":"ORDERS_EAST", "seq":5} +``` + +After a short while all streams hold the same data. + +``` +╭───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│ Stream Report │ +├────────────────┬─────────┬──────────────────────┬───────────┬──────────┬───────┬──────┬─────────┬─────────────────────────────────────┤ +│ Stream │ Storage │ Placement │ Consumers │ Messages │ Bytes │ Lost │ Deleted │ Replicas │ +├────────────────┼─────────┼──────────────────────┼───────────┼──────────┼───────┼──────┼─────────┼─────────────────────────────────────┤ +│ GLOBAL │ File │ │ 0 │ 0 │ 0 B │ 0 │ 0 │ n1-central, n2-west*, n3-east │ +│ ORDERS_CENTRAL │ File │ tags: region:central │ 0 │ 5 │ 399 B │ 0 │ 0 │ n1-central, n2-central*, n3-central │ +│ ORDERS_EAST │ File │ tags: region:east │ 0 │ 5 │ 405 B │ 0 │ 0 │ n1-east*, n2-east, n3-east │ +│ ORDERS_WEST │ File │ tags: region:west │ 0 │ 5 │ 456 B │ 0 │ 0 │ n1-west*, n2-west, n3-west │ +╰────────────────┴─────────┴──────────────────────┴───────────┴──────────┴───────┴──────┴─────────┴─────────────────────────────────────╯ +``` + +This allows portable publishers to be built, consumers will need to know the region they are in and bind to the correct stream. + + diff --git a/examples/topologies/multi-region/cli/configs/cluster-central.conf b/examples/topologies/multi-region/cli/configs/cluster-central.conf new file mode 100644 index 00000000..f3953dab --- /dev/null +++ b/examples/topologies/multi-region/cli/configs/cluster-central.conf @@ -0,0 +1,56 @@ +port: 4222 +monitor_port: 8222 +server_name: $NAME +client_advertise: $ADVERTISE + +server_tags: [$GATEWAY, $REGION] + +cluster { + port: 6222 + + routes = [ + nats-route://n1-east:6222 + nats-route://n2-east:6222 + nats-route://n3-east:6222 + nats-route://n1-west:6222 + nats-route://n2-west:6222 + nats-route://n3-west:6222 + nats-route://n1-central:6222 + nats-route://n2-central:6222 + nats-route://n3-central:6222 + ] +} + +leafnodes { + port: 7422 +} + +gateway { + name: $GATEWAY + port: 7222 +} + +jetstream { + store_dir: /data + unique_tag: "region:" +} + +accounts { + one: { + jetstream: enabled + users = [ + {user: one, password: secret} + ] + mappings: { + js.in.orders: js.in.orders_central + } + } + + system: { + users = [ + {user: system, password: secret} + ] + } +} + +system_account: system diff --git a/examples/topologies/multi-region/cli/configs/cluster-east.conf b/examples/topologies/multi-region/cli/configs/cluster-east.conf new file mode 100644 index 00000000..2df5022f --- /dev/null +++ b/examples/topologies/multi-region/cli/configs/cluster-east.conf @@ -0,0 +1,57 @@ +port: 4222 +monitor_port: 8222 +server_name: $NAME +client_advertise: $ADVERTISE + +server_tags: [$GATEWAY, $REGION] + +cluster { + port: 6222 + + routes = [ + nats-route://n1-east:6222 + nats-route://n2-east:6222 + nats-route://n3-east:6222 + nats-route://n1-west:6222 + nats-route://n2-west:6222 + nats-route://n3-west:6222 + nats-route://n1-central:6222 + nats-route://n2-central:6222 + nats-route://n3-central:6222 + ] +} + +leafnodes { + port: 7422 +} + +gateway { + name: $GATEWAY + port: 7222 +} + +jetstream { + store_dir: /data + unique_tag: "region:" +} + +accounts { + one: { + jetstream: enabled + users = [ + {user: one, password: secret} + ] + + mappings: { + js.in.orders: js.in.orders_east + } + } + + system: { + users = [ + {user: system, password: secret} + ] + } +} + +system_account: system diff --git a/examples/topologies/multi-region/cli/configs/cluster-west.conf b/examples/topologies/multi-region/cli/configs/cluster-west.conf new file mode 100644 index 00000000..5e819afe --- /dev/null +++ b/examples/topologies/multi-region/cli/configs/cluster-west.conf @@ -0,0 +1,56 @@ +port: 4222 +monitor_port: 8222 +server_name: $NAME +client_advertise: $ADVERTISE + +server_tags: [$GATEWAY, $REGION] + +cluster { + port: 6222 + + routes = [ + nats-route://n1-east:6222 + nats-route://n2-east:6222 + nats-route://n3-east:6222 + nats-route://n1-west:6222 + nats-route://n2-west:6222 + nats-route://n3-west:6222 + nats-route://n1-central:6222 + nats-route://n2-central:6222 + nats-route://n3-central:6222 + ] +} + +leafnodes { + port: 7422 +} + +gateway { + name: $GATEWAY + port: 7222 +} + +jetstream { + store_dir: /data + unique_tag: "region:" +} + +accounts { + one: { + jetstream: enabled + users = [ + {user: one, password: secret} + ] + mappings: { + js.in.orders: js.in.orders_west + } + } + + system: { + users = [ + {user: system, password: secret} + ] + } +} + +system_account: system diff --git a/examples/topologies/multi-region/cli/docker-compose.yaml b/examples/topologies/multi-region/cli/docker-compose.yaml new file mode 100644 index 00000000..bb417868 --- /dev/null +++ b/examples/topologies/multi-region/cli/docker-compose.yaml @@ -0,0 +1,143 @@ +--- +version: '3' +services: + n1.east.example.net: + container_name: n1-east + image: nats:2.9.15 + dns_search: example.net + environment: + GATEWAY: c1 + NAME: n1-east + ADVERTISE: localhost:10000 + REGION: region:east + volumes: + - "./configs/cluster-east.conf:/nats-server.conf" + ports: + - 10000:4222 + - 10100:7422 + n2.east.example.net: + container_name: n2-east + image: nats:2.9.15 + dns_search: example.net + labels: + com.docker-tc.enabled: '1' + environment: + GATEWAY: c1 + NAME: n2-east + ADVERTISE: localhost:10001 + REGION: region:east + volumes: + - "./configs/cluster-east.conf:/nats-server.conf" + ports: + - 10001:4222 + - 10101:7422 + n3.east.example.net: + container_name: n3-east + image: nats:2.9.15 + dns_search: example.net + environment: + GATEWAY: c1 + NAME: n3-east + ADVERTISE: localhost:10002 + REGION: region:east + volumes: + - "./configs/cluster-east.conf:/nats-server.conf" + ports: + - 10002:4222 + - 10102:7422 + n1.central.example.net: + container_name: n1-central + image: nats:2.9.15 + dns_search: example.net + environment: + GATEWAY: c1 + NAME: n1-central + ADVERTISE: localhost:10003 + REGION: region:central + volumes: + - "./configs/cluster-central.conf:/nats-server.conf" + ports: + - 10003:4222 + - 10103:7422 + n2.central.example.net: + container_name: n2-central + image: nats:2.9.15 + dns_search: example.net + environment: + GATEWAY: c1 + NAME: n2-central + ADVERTISE: localhost:10004 + REGION: region:central + volumes: + - "./configs/cluster-central.conf:/nats-server.conf" + ports: + - 10004:4222 + - 10104:7422 + n3.central.example.net: + container_name: n3-central + image: nats:2.9.15 + dns_search: example.net + environment: + GATEWAY: c1 + NAME: n3-central + ADVERTISE: localhost:10005 + REGION: region:central + volumes: + - "./configs/cluster-central.conf:/nats-server.conf" + ports: + - 10005:4222 + - 10105:7422 + n1.west.example.net: + container_name: n1-west + image: nats:2.9.15 + dns_search: example.net + environment: + GATEWAY: c1 + NAME: n1-west + ADVERTISE: localhost:10006 + REGION: region:west + volumes: + - "./configs/cluster-west.conf:/nats-server.conf" + ports: + - 10006:4222 + - 10106:7422 + n2.west.example.net: + container_name: n2-west + image: nats:2.9.15 + dns_search: example.net + environment: + GATEWAY: c1 + NAME: n2-west + ADVERTISE: localhost:10007 + REGION: region:west + volumes: + - "./configs/cluster-west.conf:/nats-server.conf" + ports: + - 10007:4222 + - 10107:7422 + n3.west.example.net: + container_name: n3-west + image: nats:2.9.15 + dns_search: example.net + environment: + GATEWAY: c1 + NAME: n3-west + ADVERTISE: localhost:10008 + REGION: region:west + volumes: + - "./configs/cluster-west.conf:/nats-server.conf" + ports: + - 10008:4222 + - 10108:7422 + app: + image: ${IMAGE_TAG} + depends_on: + - n1.east.example.net + - n2.east.example.net + - n3.east.example.net + - n1.west.example.net + - n2.west.example.net + - n3.west.example.net + - n1.central.example.net + - n2.central.example.net + - n3.central.example.net diff --git a/examples/topologies/multi-region/cli/main.sh b/examples/topologies/multi-region/cli/main.sh new file mode 100644 index 00000000..676ab93b --- /dev/null +++ b/examples/topologies/multi-region/cli/main.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +set -euo pipefail + +sleep 3 + +# Save a few contexts. +nats context save east-sys \ + --server nats://n1.east.example.net:4222 \ + --user system \ + --password secret + +nats context save east \ + --server nats://n1.east.example.net:4222 \ + --user one \ + --password secret + +nats context save west \ + --server nats://n1.west.example.net:4222 \ + --user one \ + --password secret + +nats context save central \ + --server nats://n1.central.example.net:4222 \ + --user one \ + --password secret + +# Report the servers. +nats --context east-sys server list + +# Creating a region-local stream requires setting a tag for the desired region. +nats --context east stream add --config /app/ORDERS_EAST.json +nats --context west stream add --config /app/ORDERS_WEST.json +nats --context central stream add --config /app/ORDERS_CENTRAL.json + +# Ensure the stream sourcing retries catch up. +sleep 5 + +# Creating a global stream involves ommitting the --tag option. +nats --context east stream add --config /app/GLOBAL.json + +# Let's see the stream report. +nats --context east stream report + +# Publish a message from a client in each region. +nats --context east req js.in.orders 1 +nats --context central req js.in.orders 1 +nats --context west req js.in.orders 1 + +# Publish a message to the global stream. +nats --context east req js.in.global.orders 1 + +# Let's see the stream report again. +nats --context east stream report