From 8014c81a708dec161875c0d894d09e306948d66b Mon Sep 17 00:00:00 2001 From: Byron Ruth Date: Fri, 24 Mar 2023 14:15:42 -0400 Subject: [PATCH 1/3] Add multi-region example Signed-off-by: Byron Ruth --- .../topologies/multi-region/cli/Dockerfile | 23 +++ .../topologies/multi-region/cli/GLOBAL.json | 23 +++ .../multi-region/cli/ORDERS_CENTRAL.json | 39 ++++ .../multi-region/cli/ORDERS_EAST.json | 39 ++++ .../multi-region/cli/ORDERS_WEST.json | 39 ++++ .../topologies/multi-region/cli/README.md | 107 ++++++++++ .../cli/configs/cluster-central.conf | 56 ++++++ .../cli/configs/cluster-east.conf | 57 ++++++ .../cli/configs/cluster-west.conf | 56 ++++++ .../cli/contexts/central-system.json | 20 ++ .../cli/contexts/central-user.json | 20 ++ .../cli/contexts/east-system.json | 20 ++ .../multi-region/cli/contexts/east-user.json | 20 ++ .../cli/contexts/west-system.json | 20 ++ .../multi-region/cli/contexts/west-user.json | 20 ++ .../multi-region/cli/docker-compose.yaml | 185 ++++++++++++++++++ examples/topologies/multi-region/cli/main.sh | 44 +++++ 17 files changed, 788 insertions(+) create mode 100644 examples/topologies/multi-region/cli/Dockerfile create mode 100644 examples/topologies/multi-region/cli/GLOBAL.json create mode 100644 examples/topologies/multi-region/cli/ORDERS_CENTRAL.json create mode 100644 examples/topologies/multi-region/cli/ORDERS_EAST.json create mode 100644 examples/topologies/multi-region/cli/ORDERS_WEST.json create mode 100644 examples/topologies/multi-region/cli/README.md create mode 100644 examples/topologies/multi-region/cli/configs/cluster-central.conf create mode 100644 examples/topologies/multi-region/cli/configs/cluster-east.conf create mode 100644 examples/topologies/multi-region/cli/configs/cluster-west.conf create mode 100644 examples/topologies/multi-region/cli/contexts/central-system.json create mode 100644 examples/topologies/multi-region/cli/contexts/central-user.json create mode 100644 examples/topologies/multi-region/cli/contexts/east-system.json create mode 100644 examples/topologies/multi-region/cli/contexts/east-user.json create mode 100644 examples/topologies/multi-region/cli/contexts/west-system.json create mode 100644 examples/topologies/multi-region/cli/contexts/west-user.json create mode 100644 examples/topologies/multi-region/cli/docker-compose.yaml create mode 100644 examples/topologies/multi-region/cli/main.sh diff --git a/examples/topologies/multi-region/cli/Dockerfile b/examples/topologies/multi-region/cli/Dockerfile new file mode 100644 index 00000000..99f733cb --- /dev/null +++ b/examples/topologies/multi-region/cli/Dockerfile @@ -0,0 +1,23 @@ +FROM golang:1.19-alpine3.17 AS build + +RUN apk update && apk add git + +RUN go install github.com/nats-io/nats-server/v2@v2.9.15 +RUN go install github.com/nats-io/natscli/nats@main +RUN go install github.com/nats-io/nsc/v2@v2.7.8 + +FROM alpine:3.17 + +RUN apk add bash curl + +COPY --from=build /go/bin/nats-server /usr/local/bin/ +COPY --from=build /go/bin/nats /usr/local/bin/ +COPY --from=build /go/bin/nsc /usr/local/bin/ + +WORKDIR /app + +COPY . . + +ENTRYPOINT ["bash"] + +CMD ["main.sh"] diff --git a/examples/topologies/multi-region/cli/GLOBAL.json b/examples/topologies/multi-region/cli/GLOBAL.json new file mode 100644 index 00000000..773321fa --- /dev/null +++ b/examples/topologies/multi-region/cli/GLOBAL.json @@ -0,0 +1,23 @@ +{ + "name": "GLOBAL", + "subjects": [ + "js.in.global.>" + ], + "retention": "limits", + "max_consumers": -1, + "max_msgs_per_subject": -1, + "max_msgs": -1, + "max_bytes": -1, + "max_age": 0, + "max_msg_size": -1, + "storage": "file", + "discard": "old", + "num_replicas": 3, + "duplicate_window": 120000000000, + "sealed": false, + "deny_delete": false, + "deny_purge": false, + "allow_rollup_hdrs": false, + "allow_direct": false, + "mirror_direct": false +} diff --git a/examples/topologies/multi-region/cli/ORDERS_CENTRAL.json b/examples/topologies/multi-region/cli/ORDERS_CENTRAL.json new file mode 100644 index 00000000..b2cd4c20 --- /dev/null +++ b/examples/topologies/multi-region/cli/ORDERS_CENTRAL.json @@ -0,0 +1,39 @@ +{ + "name": "ORDERS_CENTRAL", + "subjects": [ + "js.in.orders_central" + ], + "retention": "limits", + "max_consumers": -1, + "max_msgs_per_subject": -1, + "max_msgs": -1, + "max_bytes": -1, + "max_age": 0, + "max_msg_size": -1, + "storage": "file", + "discard": "old", + "num_replicas": 3, + "duplicate_window": 120000000000, + "placement": { + "cluster": "", + "tags": [ + "region:central" + ] + }, + "sources": [ + { + "name": "ORDERS_EAST", + "filter_subject": "js.in.orders_east" + }, + { + "name": "ORDERS_WEST", + "filter_subject": "js.in.orders_west" + } + ], + "sealed": false, + "deny_delete": false, + "deny_purge": false, + "allow_rollup_hdrs": false, + "allow_direct": false, + "mirror_direct": false +} diff --git a/examples/topologies/multi-region/cli/ORDERS_EAST.json b/examples/topologies/multi-region/cli/ORDERS_EAST.json new file mode 100644 index 00000000..33584ee8 --- /dev/null +++ b/examples/topologies/multi-region/cli/ORDERS_EAST.json @@ -0,0 +1,39 @@ +{ + "name": "ORDERS_EAST", + "subjects": [ + "js.in.orders_east" + ], + "retention": "limits", + "max_consumers": -1, + "max_msgs_per_subject": -1, + "max_msgs": -1, + "max_bytes": -1, + "max_age": 0, + "max_msg_size": -1, + "storage": "file", + "discard": "old", + "num_replicas": 3, + "duplicate_window": 120000000000, + "placement": { + "cluster": "", + "tags": [ + "region:east" + ] + }, + "sources": [ + { + "name": "ORDERS_WEST", + "filter_subject": "js.in.orders_west" + }, + { + "name": "ORDERS_CENTRAL", + "filter_subject": "js.in.orders_central" + } + ], + "sealed": false, + "deny_delete": false, + "deny_purge": false, + "allow_rollup_hdrs": false, + "allow_direct": false, + "mirror_direct": false +} diff --git a/examples/topologies/multi-region/cli/ORDERS_WEST.json b/examples/topologies/multi-region/cli/ORDERS_WEST.json new file mode 100644 index 00000000..db6c0c29 --- /dev/null +++ b/examples/topologies/multi-region/cli/ORDERS_WEST.json @@ -0,0 +1,39 @@ +{ + "name": "ORDERS_WEST", + "subjects": [ + "js.in.orders_west" + ], + "retention": "limits", + "max_consumers": -1, + "max_msgs_per_subject": -1, + "max_msgs": -1, + "max_bytes": -1, + "max_age": 0, + "max_msg_size": -1, + "storage": "file", + "discard": "old", + "num_replicas": 3, + "duplicate_window": 120000000000, + "placement": { + "cluster": "", + "tags": [ + "region:west" + ] + }, + "sources": [ + { + "name": "ORDERS_EAST", + "filter_subject": "js.in.orders_east" + }, + { + "name": "ORDERS_CENTRAL", + "filter_subject": "js.in.orders_central" + } + ], + "sealed": false, + "deny_delete": false, + "deny_purge": false, + "allow_rollup_hdrs": false, + "allow_direct": false, + "mirror_direct": false +} diff --git a/examples/topologies/multi-region/cli/README.md b/examples/topologies/multi-region/cli/README.md new file mode 100644 index 00000000..c0d209d6 --- /dev/null +++ b/examples/topologies/multi-region/cli/README.md @@ -0,0 +1,107 @@ +# Multi Region NATS Cluster + +This demonstrates a very basic NATS Cluster using only routes across 3 regions. + +This should be used in cases where a multi region setup is needed and one requires a stream to be globally deployed and have globally consistent message deduplication. + +## Constraints + + * This should be deployed in networks with generally below 100ms latency. For example in GCP using Tier-1 network connectivity in US east, west and central. + * Streams should generally be R3 when stretched out of a single region to mitigate the big exposure they would have to latency if R5 + * Multi-region streams should not be the default, ideally these are only used for those cases where global deduplication is needed + * For general streams where eventual consistency is acceptible streams should be bound to a region and replicated into others if desired + +## Configuration guidelines + +### Server Tags + +Each server is tagged with a regional tag, for example, `region:east` or `region:west`. The servers are configured with anti-affinity on this tag using the following configuration: + +``` +jetstream { + unique_tag: "region:" +} +``` + +This ensures that streams that are not specifically bound to a single region will be split across the 3 regions. + +### Configuring single region streams + +``` +$ nats stream add --tag region:east --replicas 3 EAST +... +Cluster Information: + + Name: c1 + Leader: n1-east + Replica: n2-east, current, seen 1ms ago + Replica: n3-east, current, seen 0s ago +``` + +Note the servers are `n1-east`, `n2-east`, `n3-east`. + +### Configuring a global stream + +``` +$ nats stream add --replicas 3 GLOBAL +... +Cluster Information: + + Name: c1 + Leader: n1-central + Replica: n2-east, current, seen 0s ago + Replica: n3-west, current, seen 0s ago + +``` + +Note here we give no placement directive so the server will spread it across regions due to the `unique_tag` setting. Servers are `n1-central`, `n2-east` and `n3-west` - 1 per region. + +### Configuring replicated streams + +To facilitate an eventually consistent but single config set up we show how to create 3 streams: + + * `ORDERS_EAST` listening on subjects `js.in.orders_east` + * `ORDERS_WEST` listening on subjects `js.in.orders_west` + * `ORDERS_CENTRAL` listening on subjects `js.in.orders.central` + +We then use server mappings in each region to map `js.in.orders` to the in-region subject: + +``` +accounts { + one: { + jetstream: enabled + mappings: { + js.in.orders: js.in.orders_central + } + } +} +``` + +We can now publish to one subject and depend on which region we connect to the local stream will handle - then replicate - the data: + +``` +$ nats --context contexts/central-user.json req js.in.orders 1 +{"stream":"ORDERS_CENTRAL", "seq":4} + +$ nats --context contexts/east-user.json req js.in.orders 1 +{"stream":"ORDERS_EAST", "seq":5} +``` + +After a short while all streams hold the same data. + +``` +╭───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│ Stream Report │ +├────────────────┬─────────┬──────────────────────┬───────────┬──────────┬───────┬──────┬─────────┬─────────────────────────────────────┤ +│ Stream │ Storage │ Placement │ Consumers │ Messages │ Bytes │ Lost │ Deleted │ Replicas │ +├────────────────┼─────────┼──────────────────────┼───────────┼──────────┼───────┼──────┼─────────┼─────────────────────────────────────┤ +│ GLOBAL │ File │ │ 0 │ 0 │ 0 B │ 0 │ 0 │ n1-central, n2-west*, n3-east │ +│ ORDERS_CENTRAL │ File │ tags: region:central │ 0 │ 5 │ 399 B │ 0 │ 0 │ n1-central, n2-central*, n3-central │ +│ ORDERS_EAST │ File │ tags: region:east │ 0 │ 5 │ 405 B │ 0 │ 0 │ n1-east*, n2-east, n3-east │ +│ ORDERS_WEST │ File │ tags: region:west │ 0 │ 5 │ 456 B │ 0 │ 0 │ n1-west*, n2-west, n3-west │ +╰────────────────┴─────────┴──────────────────────┴───────────┴──────────┴───────┴──────┴─────────┴─────────────────────────────────────╯ +``` + +This allows portable publishers to be built, consumers will need to know the region they are in and bind to the correct stream. + + diff --git a/examples/topologies/multi-region/cli/configs/cluster-central.conf b/examples/topologies/multi-region/cli/configs/cluster-central.conf new file mode 100644 index 00000000..f3953dab --- /dev/null +++ b/examples/topologies/multi-region/cli/configs/cluster-central.conf @@ -0,0 +1,56 @@ +port: 4222 +monitor_port: 8222 +server_name: $NAME +client_advertise: $ADVERTISE + +server_tags: [$GATEWAY, $REGION] + +cluster { + port: 6222 + + routes = [ + nats-route://n1-east:6222 + nats-route://n2-east:6222 + nats-route://n3-east:6222 + nats-route://n1-west:6222 + nats-route://n2-west:6222 + nats-route://n3-west:6222 + nats-route://n1-central:6222 + nats-route://n2-central:6222 + nats-route://n3-central:6222 + ] +} + +leafnodes { + port: 7422 +} + +gateway { + name: $GATEWAY + port: 7222 +} + +jetstream { + store_dir: /data + unique_tag: "region:" +} + +accounts { + one: { + jetstream: enabled + users = [ + {user: one, password: secret} + ] + mappings: { + js.in.orders: js.in.orders_central + } + } + + system: { + users = [ + {user: system, password: secret} + ] + } +} + +system_account: system diff --git a/examples/topologies/multi-region/cli/configs/cluster-east.conf b/examples/topologies/multi-region/cli/configs/cluster-east.conf new file mode 100644 index 00000000..2df5022f --- /dev/null +++ b/examples/topologies/multi-region/cli/configs/cluster-east.conf @@ -0,0 +1,57 @@ +port: 4222 +monitor_port: 8222 +server_name: $NAME +client_advertise: $ADVERTISE + +server_tags: [$GATEWAY, $REGION] + +cluster { + port: 6222 + + routes = [ + nats-route://n1-east:6222 + nats-route://n2-east:6222 + nats-route://n3-east:6222 + nats-route://n1-west:6222 + nats-route://n2-west:6222 + nats-route://n3-west:6222 + nats-route://n1-central:6222 + nats-route://n2-central:6222 + nats-route://n3-central:6222 + ] +} + +leafnodes { + port: 7422 +} + +gateway { + name: $GATEWAY + port: 7222 +} + +jetstream { + store_dir: /data + unique_tag: "region:" +} + +accounts { + one: { + jetstream: enabled + users = [ + {user: one, password: secret} + ] + + mappings: { + js.in.orders: js.in.orders_east + } + } + + system: { + users = [ + {user: system, password: secret} + ] + } +} + +system_account: system diff --git a/examples/topologies/multi-region/cli/configs/cluster-west.conf b/examples/topologies/multi-region/cli/configs/cluster-west.conf new file mode 100644 index 00000000..5e819afe --- /dev/null +++ b/examples/topologies/multi-region/cli/configs/cluster-west.conf @@ -0,0 +1,56 @@ +port: 4222 +monitor_port: 8222 +server_name: $NAME +client_advertise: $ADVERTISE + +server_tags: [$GATEWAY, $REGION] + +cluster { + port: 6222 + + routes = [ + nats-route://n1-east:6222 + nats-route://n2-east:6222 + nats-route://n3-east:6222 + nats-route://n1-west:6222 + nats-route://n2-west:6222 + nats-route://n3-west:6222 + nats-route://n1-central:6222 + nats-route://n2-central:6222 + nats-route://n3-central:6222 + ] +} + +leafnodes { + port: 7422 +} + +gateway { + name: $GATEWAY + port: 7222 +} + +jetstream { + store_dir: /data + unique_tag: "region:" +} + +accounts { + one: { + jetstream: enabled + users = [ + {user: one, password: secret} + ] + mappings: { + js.in.orders: js.in.orders_west + } + } + + system: { + users = [ + {user: system, password: secret} + ] + } +} + +system_account: system diff --git a/examples/topologies/multi-region/cli/contexts/central-system.json b/examples/topologies/multi-region/cli/contexts/central-system.json new file mode 100644 index 00000000..50ff18cc --- /dev/null +++ b/examples/topologies/multi-region/cli/contexts/central-system.json @@ -0,0 +1,20 @@ +{ + "description": "", + "url": "nats://localhost:10003,nats://localhost:10004,nats://localhost:10005", + "socks_proxy": "", + "token": "system", + "user": "system", + "password": "secret", + "creds": "", + "nkey": "", + "cert": "", + "key": "", + "ca": "", + "nsc": "", + "jetstream_domain": "", + "jetstream_api_prefix": "", + "jetstream_event_prefix": "", + "inbox_prefix": "", + "user_jwt": "", + "color_scheme": "red" +} diff --git a/examples/topologies/multi-region/cli/contexts/central-user.json b/examples/topologies/multi-region/cli/contexts/central-user.json new file mode 100644 index 00000000..7b9349dd --- /dev/null +++ b/examples/topologies/multi-region/cli/contexts/central-user.json @@ -0,0 +1,20 @@ +{ + "description": "", + "url": "nats://localhost:10003,nats://localhost:10004,nats://localhost:10005", + "socks_proxy": "", + "token": "one", + "user": "one", + "password": "secret", + "creds": "", + "nkey": "", + "cert": "", + "key": "", + "ca": "", + "nsc": "", + "jetstream_domain": "", + "jetstream_api_prefix": "", + "jetstream_event_prefix": "", + "inbox_prefix": "", + "user_jwt": "", + "color_scheme": "yellow" +} diff --git a/examples/topologies/multi-region/cli/contexts/east-system.json b/examples/topologies/multi-region/cli/contexts/east-system.json new file mode 100644 index 00000000..a0aa798e --- /dev/null +++ b/examples/topologies/multi-region/cli/contexts/east-system.json @@ -0,0 +1,20 @@ +{ + "description": "", + "url": "nats://localhost:10000,nats://localhost:10001,nats://localhost:10002", + "socks_proxy": "", + "token": "system", + "user": "system", + "password": "secret", + "creds": "", + "nkey": "", + "cert": "", + "key": "", + "ca": "", + "nsc": "", + "jetstream_domain": "", + "jetstream_api_prefix": "", + "jetstream_event_prefix": "", + "inbox_prefix": "", + "user_jwt": "", + "color_scheme": "red" +} diff --git a/examples/topologies/multi-region/cli/contexts/east-user.json b/examples/topologies/multi-region/cli/contexts/east-user.json new file mode 100644 index 00000000..422a3f38 --- /dev/null +++ b/examples/topologies/multi-region/cli/contexts/east-user.json @@ -0,0 +1,20 @@ +{ + "description": "", + "url": "nats://localhost:10000,nats://localhost:10001,nats://localhost:10002", + "socks_proxy": "", + "token": "one", + "user": "one", + "password": "secret", + "creds": "", + "nkey": "", + "cert": "", + "key": "", + "ca": "", + "nsc": "", + "jetstream_domain": "", + "jetstream_api_prefix": "", + "jetstream_event_prefix": "", + "inbox_prefix": "", + "user_jwt": "", + "color_scheme": "magenta" +} diff --git a/examples/topologies/multi-region/cli/contexts/west-system.json b/examples/topologies/multi-region/cli/contexts/west-system.json new file mode 100644 index 00000000..6a18c5cd --- /dev/null +++ b/examples/topologies/multi-region/cli/contexts/west-system.json @@ -0,0 +1,20 @@ +{ + "description": "", + "url": "nats://localhost:10006,nats://localhost:10007,nats://localhost:10008", + "socks_proxy": "", + "token": "system", + "user": "system", + "password": "secret", + "creds": "", + "nkey": "", + "cert": "", + "key": "", + "ca": "", + "nsc": "", + "jetstream_domain": "", + "jetstream_api_prefix": "", + "jetstream_event_prefix": "", + "inbox_prefix": "", + "user_jwt": "", + "color_scheme": "red" +} diff --git a/examples/topologies/multi-region/cli/contexts/west-user.json b/examples/topologies/multi-region/cli/contexts/west-user.json new file mode 100644 index 00000000..d3c51eba --- /dev/null +++ b/examples/topologies/multi-region/cli/contexts/west-user.json @@ -0,0 +1,20 @@ +{ + "description": "", + "url": "nats://localhost:10006,nats://localhost:10007,nats://localhost:10008", + "socks_proxy": "", + "token": "one", + "user": "one", + "password": "secret", + "creds": "", + "nkey": "", + "cert": "", + "key": "", + "ca": "", + "nsc": "", + "jetstream_domain": "", + "jetstream_api_prefix": "", + "jetstream_event_prefix": "", + "inbox_prefix": "", + "user_jwt": "", + "color_scheme": "green" +} diff --git a/examples/topologies/multi-region/cli/docker-compose.yaml b/examples/topologies/multi-region/cli/docker-compose.yaml new file mode 100644 index 00000000..6b712e09 --- /dev/null +++ b/examples/topologies/multi-region/cli/docker-compose.yaml @@ -0,0 +1,185 @@ +--- +version: '3' +services: + n1.east.example.net: + container_name: n1-east + image: nats + dns_search: example.net + environment: + GATEWAY: c1 + NAME: n1-east + ADVERTISE: localhost:10000 + REGION: region:east + networks: + - nats-cluster1 + - shared + volumes: + - "./configs/cluster-east.conf:/nats-server.conf" + - "./data/n1-east:/data" + ports: + - 10000:4222 + - 10100:7422 + n2.east.example.net: + container_name: n2-east + image: nats + dns_search: example.net + labels: + com.docker-tc.enabled: '1' + environment: + GATEWAY: c1 + NAME: n2-east + ADVERTISE: localhost:10001 + REGION: region:east + networks: + - nats-cluster1 + - shared + volumes: + - "./configs/cluster-east.conf:/nats-server.conf" + - "./data/n2-east:/data" + ports: + - 10001:4222 + - 10101:7422 + n3.east.example.net: + container_name: n3-east + image: nats + dns_search: example.net + environment: + GATEWAY: c1 + NAME: n3-east + ADVERTISE: localhost:10002 + REGION: region:east + networks: + - nats-cluster1 + - shared + volumes: + - "./configs/cluster-east.conf:/nats-server.conf" + - "./data/n3-east:/data" + ports: + - 10002:4222 + - 10102:7422 + n1.central.example.net: + container_name: n1-central + image: nats + dns_search: example.net + environment: + GATEWAY: c1 + NAME: n1-central + ADVERTISE: localhost:10003 + REGION: region:central + networks: + - nats-cluster1 + - shared + volumes: + - "./configs/cluster-central.conf:/nats-server.conf" + - "./data/n1-central:/data" + ports: + - 10003:4222 + - 10103:7422 + n2.central.example.net: + container_name: n2-central + image: nats + dns_search: example.net + environment: + GATEWAY: c1 + NAME: n2-central + ADVERTISE: localhost:10004 + REGION: region:central + networks: + - nats-cluster1 + - shared + volumes: + - "./configs/cluster-central.conf:/nats-server.conf" + - "./data/n2-central:/data" + ports: + - 10004:4222 + - 10104:7422 + n3.central.example.net: + container_name: n3-central + image: nats + dns_search: example.net + environment: + GATEWAY: c1 + NAME: n3-central + ADVERTISE: localhost:10005 + REGION: region:central + networks: + - nats-cluster1 + - shared + volumes: + - "./configs/cluster-central.conf:/nats-server.conf" + - "./data/n3-central:/data" + ports: + - 10005:4222 + - 10105:7422 + n1.west.example.net: + container_name: n1-west + image: nats + dns_search: example.net + environment: + GATEWAY: c1 + NAME: n1-west + ADVERTISE: localhost:10006 + REGION: region:west + networks: + - nats-cluster1 + - shared + volumes: + - "./configs/cluster-west.conf:/nats-server.conf" + - "./data/n1-west:/data" + ports: + - 10006:4222 + - 10106:7422 + n2.west.example.net: + container_name: n2-west + image: nats + dns_search: example.net + environment: + GATEWAY: c1 + NAME: n2-west + ADVERTISE: localhost:10007 + REGION: region:west + networks: + - nats-cluster1 + - shared + volumes: + - "./configs/cluster-west.conf:/nats-server.conf" + - "./data/n2-west:/data" + ports: + - 10007:4222 + - 10107:7422 + n3.west.example.net: + container_name: n3-west + image: nats + dns_search: example.net + environment: + GATEWAY: c1 + NAME: n3-west + ADVERTISE: localhost:10008 + REGION: region:west + networks: + - nats-cluster1 + - shared + volumes: + - "./configs/cluster-west.conf:/nats-server.conf" + - "./data/n3-west:/data" + ports: + - 10008:4222 + - 10108:7422 + app: + image: ${IMAGE_TAG} + depends_on: + - n1.east.example.net + - n2.east.example.net + - n3.east.example.net + - n1.west.example.net + - n2.west.example.net + - n3.west.example.net + - n1.central.example.net + - n2.central.example.net + - n3.central.example.net + networks: + - shared + +networks: + shared: {} + nats-cluster1: {} diff --git a/examples/topologies/multi-region/cli/main.sh b/examples/topologies/multi-region/cli/main.sh new file mode 100644 index 00000000..bff40e97 --- /dev/null +++ b/examples/topologies/multi-region/cli/main.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +set -euo pipefail + +sleep 3 + +nats context save east \ + --server nats://n1.east.example.net:4222 \ + --user one \ + --password secret + +nats context save west \ + --server nats://n1.west.example.net:4222 \ + --user one \ + --password secret + +nats context save central \ + --server nats://n1.central.example.net:4222 \ + --user one \ + --password secret + +# Creating a region-local stream requires setting a tag for the desired region. +nats --context east stream add --config /app/ORDERS_EAST.json +nats --context west stream add --config /app/ORDERS_WEST.json +nats --context central stream add --config /app/ORDERS_CENTRAL.json + +# Creating a global stream involves ommitting the --tag option. +nats --context east stream add --config /app/GLOBAL.json + +# Let's see the stream report. +nats --context east stream report + +# Publish a message from a client in each region. +nats --context east req js.in.orders 1 +nats --context west req js.in.orders 1 +nats --context central req js.in.orders 1 + +# Publish a message to the global stream. +nats --context east req js.in.global.orders 1 + +sleep 1 + +# Let's see the stream report again. +nats --context east stream report From 351f73af4fede6e6adbac346f458dbd7caee552e Mon Sep 17 00:00:00 2001 From: Byron Ruth Date: Mon, 27 Mar 2023 14:06:19 -0400 Subject: [PATCH 2/3] Ensure streams are created before source edits Signed-off-by: Byron Ruth --- .../cli/contexts/central-system.json | 20 ----- .../cli/contexts/central-user.json | 20 ----- .../cli/contexts/east-system.json | 20 ----- .../multi-region/cli/contexts/east-user.json | 20 ----- .../cli/contexts/west-system.json | 20 ----- .../multi-region/cli/contexts/west-user.json | 20 ----- .../multi-region/cli/docker-compose.yaml | 78 +++++-------------- examples/topologies/multi-region/cli/main.sh | 25 +++++- 8 files changed, 39 insertions(+), 184 deletions(-) delete mode 100644 examples/topologies/multi-region/cli/contexts/central-system.json delete mode 100644 examples/topologies/multi-region/cli/contexts/central-user.json delete mode 100644 examples/topologies/multi-region/cli/contexts/east-system.json delete mode 100644 examples/topologies/multi-region/cli/contexts/east-user.json delete mode 100644 examples/topologies/multi-region/cli/contexts/west-system.json delete mode 100644 examples/topologies/multi-region/cli/contexts/west-user.json diff --git a/examples/topologies/multi-region/cli/contexts/central-system.json b/examples/topologies/multi-region/cli/contexts/central-system.json deleted file mode 100644 index 50ff18cc..00000000 --- a/examples/topologies/multi-region/cli/contexts/central-system.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "description": "", - "url": "nats://localhost:10003,nats://localhost:10004,nats://localhost:10005", - "socks_proxy": "", - "token": "system", - "user": "system", - "password": "secret", - "creds": "", - "nkey": "", - "cert": "", - "key": "", - "ca": "", - "nsc": "", - "jetstream_domain": "", - "jetstream_api_prefix": "", - "jetstream_event_prefix": "", - "inbox_prefix": "", - "user_jwt": "", - "color_scheme": "red" -} diff --git a/examples/topologies/multi-region/cli/contexts/central-user.json b/examples/topologies/multi-region/cli/contexts/central-user.json deleted file mode 100644 index 7b9349dd..00000000 --- a/examples/topologies/multi-region/cli/contexts/central-user.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "description": "", - "url": "nats://localhost:10003,nats://localhost:10004,nats://localhost:10005", - "socks_proxy": "", - "token": "one", - "user": "one", - "password": "secret", - "creds": "", - "nkey": "", - "cert": "", - "key": "", - "ca": "", - "nsc": "", - "jetstream_domain": "", - "jetstream_api_prefix": "", - "jetstream_event_prefix": "", - "inbox_prefix": "", - "user_jwt": "", - "color_scheme": "yellow" -} diff --git a/examples/topologies/multi-region/cli/contexts/east-system.json b/examples/topologies/multi-region/cli/contexts/east-system.json deleted file mode 100644 index a0aa798e..00000000 --- a/examples/topologies/multi-region/cli/contexts/east-system.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "description": "", - "url": "nats://localhost:10000,nats://localhost:10001,nats://localhost:10002", - "socks_proxy": "", - "token": "system", - "user": "system", - "password": "secret", - "creds": "", - "nkey": "", - "cert": "", - "key": "", - "ca": "", - "nsc": "", - "jetstream_domain": "", - "jetstream_api_prefix": "", - "jetstream_event_prefix": "", - "inbox_prefix": "", - "user_jwt": "", - "color_scheme": "red" -} diff --git a/examples/topologies/multi-region/cli/contexts/east-user.json b/examples/topologies/multi-region/cli/contexts/east-user.json deleted file mode 100644 index 422a3f38..00000000 --- a/examples/topologies/multi-region/cli/contexts/east-user.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "description": "", - "url": "nats://localhost:10000,nats://localhost:10001,nats://localhost:10002", - "socks_proxy": "", - "token": "one", - "user": "one", - "password": "secret", - "creds": "", - "nkey": "", - "cert": "", - "key": "", - "ca": "", - "nsc": "", - "jetstream_domain": "", - "jetstream_api_prefix": "", - "jetstream_event_prefix": "", - "inbox_prefix": "", - "user_jwt": "", - "color_scheme": "magenta" -} diff --git a/examples/topologies/multi-region/cli/contexts/west-system.json b/examples/topologies/multi-region/cli/contexts/west-system.json deleted file mode 100644 index 6a18c5cd..00000000 --- a/examples/topologies/multi-region/cli/contexts/west-system.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "description": "", - "url": "nats://localhost:10006,nats://localhost:10007,nats://localhost:10008", - "socks_proxy": "", - "token": "system", - "user": "system", - "password": "secret", - "creds": "", - "nkey": "", - "cert": "", - "key": "", - "ca": "", - "nsc": "", - "jetstream_domain": "", - "jetstream_api_prefix": "", - "jetstream_event_prefix": "", - "inbox_prefix": "", - "user_jwt": "", - "color_scheme": "red" -} diff --git a/examples/topologies/multi-region/cli/contexts/west-user.json b/examples/topologies/multi-region/cli/contexts/west-user.json deleted file mode 100644 index d3c51eba..00000000 --- a/examples/topologies/multi-region/cli/contexts/west-user.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "description": "", - "url": "nats://localhost:10006,nats://localhost:10007,nats://localhost:10008", - "socks_proxy": "", - "token": "one", - "user": "one", - "password": "secret", - "creds": "", - "nkey": "", - "cert": "", - "key": "", - "ca": "", - "nsc": "", - "jetstream_domain": "", - "jetstream_api_prefix": "", - "jetstream_event_prefix": "", - "inbox_prefix": "", - "user_jwt": "", - "color_scheme": "green" -} diff --git a/examples/topologies/multi-region/cli/docker-compose.yaml b/examples/topologies/multi-region/cli/docker-compose.yaml index 6b712e09..1460f157 100644 --- a/examples/topologies/multi-region/cli/docker-compose.yaml +++ b/examples/topologies/multi-region/cli/docker-compose.yaml @@ -3,25 +3,21 @@ version: '3' services: n1.east.example.net: container_name: n1-east - image: nats + image: synadia/nats-server:nightly-main dns_search: example.net environment: GATEWAY: c1 NAME: n1-east ADVERTISE: localhost:10000 REGION: region:east - networks: - - nats-cluster1 - - shared volumes: - - "./configs/cluster-east.conf:/nats-server.conf" - - "./data/n1-east:/data" + - "./configs/cluster-east.conf:/nats/conf/nats-server.conf" ports: - 10000:4222 - 10100:7422 n2.east.example.net: container_name: n2-east - image: nats + image: synadia/nats-server:nightly-main dns_search: example.net labels: com.docker-tc.enabled: '1' @@ -30,138 +26,106 @@ services: NAME: n2-east ADVERTISE: localhost:10001 REGION: region:east - networks: - - nats-cluster1 - - shared volumes: - - "./configs/cluster-east.conf:/nats-server.conf" - - "./data/n2-east:/data" + - "./configs/cluster-east.conf:/nats/conf/nats-server.conf" ports: - 10001:4222 - 10101:7422 n3.east.example.net: container_name: n3-east - image: nats + image: synadia/nats-server:nightly-main dns_search: example.net environment: GATEWAY: c1 NAME: n3-east ADVERTISE: localhost:10002 REGION: region:east - networks: - - nats-cluster1 - - shared volumes: - - "./configs/cluster-east.conf:/nats-server.conf" - - "./data/n3-east:/data" + - "./configs/cluster-east.conf:/nats/conf/nats-server.conf" ports: - 10002:4222 - 10102:7422 n1.central.example.net: container_name: n1-central - image: nats + image: synadia/nats-server:nightly-main dns_search: example.net environment: GATEWAY: c1 NAME: n1-central ADVERTISE: localhost:10003 REGION: region:central - networks: - - nats-cluster1 - - shared volumes: - - "./configs/cluster-central.conf:/nats-server.conf" - - "./data/n1-central:/data" + - "./configs/cluster-central.conf:/nats/conf/nats-server.conf" ports: - 10003:4222 - 10103:7422 n2.central.example.net: container_name: n2-central - image: nats + image: synadia/nats-server:nightly-main dns_search: example.net environment: GATEWAY: c1 NAME: n2-central ADVERTISE: localhost:10004 REGION: region:central - networks: - - nats-cluster1 - - shared volumes: - - "./configs/cluster-central.conf:/nats-server.conf" - - "./data/n2-central:/data" + - "./configs/cluster-central.conf:/nats/conf/nats-server.conf" ports: - 10004:4222 - 10104:7422 n3.central.example.net: container_name: n3-central - image: nats + image: synadia/nats-server:nightly-main dns_search: example.net environment: GATEWAY: c1 NAME: n3-central ADVERTISE: localhost:10005 REGION: region:central - networks: - - nats-cluster1 - - shared volumes: - - "./configs/cluster-central.conf:/nats-server.conf" - - "./data/n3-central:/data" + - "./configs/cluster-central.conf:/nats/conf/nats-server.conf" ports: - 10005:4222 - 10105:7422 n1.west.example.net: container_name: n1-west - image: nats + image: synadia/nats-server:nightly-main dns_search: example.net environment: GATEWAY: c1 NAME: n1-west ADVERTISE: localhost:10006 REGION: region:west - networks: - - nats-cluster1 - - shared volumes: - - "./configs/cluster-west.conf:/nats-server.conf" - - "./data/n1-west:/data" + - "./configs/cluster-west.conf:/nats/conf/nats-server.conf" ports: - 10006:4222 - 10106:7422 n2.west.example.net: container_name: n2-west - image: nats + image: synadia/nats-server:nightly-main dns_search: example.net environment: GATEWAY: c1 NAME: n2-west ADVERTISE: localhost:10007 REGION: region:west - networks: - - nats-cluster1 - - shared volumes: - - "./configs/cluster-west.conf:/nats-server.conf" - - "./data/n2-west:/data" + - "./configs/cluster-west.conf:/nats/conf/nats-server.conf" ports: - 10007:4222 - 10107:7422 n3.west.example.net: container_name: n3-west - image: nats + image: synadia/nats-server:nightly-main dns_search: example.net environment: GATEWAY: c1 NAME: n3-west ADVERTISE: localhost:10008 REGION: region:west - networks: - - nats-cluster1 - - shared volumes: - - "./configs/cluster-west.conf:/nats-server.conf" - - "./data/n3-west:/data" + - "./configs/cluster-west.conf:/nats/conf/nats-server.conf" ports: - 10008:4222 - 10108:7422 @@ -177,9 +141,3 @@ services: - n1.central.example.net - n2.central.example.net - n3.central.example.net - networks: - - shared - -networks: - shared: {} - nats-cluster1: {} diff --git a/examples/topologies/multi-region/cli/main.sh b/examples/topologies/multi-region/cli/main.sh index bff40e97..3560c716 100644 --- a/examples/topologies/multi-region/cli/main.sh +++ b/examples/topologies/multi-region/cli/main.sh @@ -4,6 +4,12 @@ set -euo pipefail sleep 3 +# Save a few contexts. +nats context save east-sys \ + --server nats://n1.east.example.net:4222 \ + --user system \ + --password secret + nats context save east \ --server nats://n1.east.example.net:4222 \ --user one \ @@ -19,10 +25,17 @@ nats context save central \ --user one \ --password secret +# Report the servers. +nats --context east-sys server list + +nats --context east stream add ORDERS_EAST --tag region:east --replicas 3 --subjects js.in.orders_east --defaults +nats --context west stream add ORDERS_WEST --tag region:west --replicas 3 --subjects js.in.orders_west --defaults +nats --context central stream add ORDERS_CENTRAL --tag region:central --replicas 3 --subjects js.in.orders_central --defaults + # Creating a region-local stream requires setting a tag for the desired region. -nats --context east stream add --config /app/ORDERS_EAST.json -nats --context west stream add --config /app/ORDERS_WEST.json -nats --context central stream add --config /app/ORDERS_CENTRAL.json +nats --context east stream edit ORDERS_EAST --force --config /app/ORDERS_EAST.json +nats --context west stream edit ORDERS_WEST --force --config /app/ORDERS_WEST.json +nats --context central stream edit ORDERS_CENTRAL --force --config /app/ORDERS_CENTRAL.json # Creating a global stream involves ommitting the --tag option. nats --context east stream add --config /app/GLOBAL.json @@ -32,8 +45,8 @@ nats --context east stream report # Publish a message from a client in each region. nats --context east req js.in.orders 1 -nats --context west req js.in.orders 1 nats --context central req js.in.orders 1 +nats --context west req js.in.orders 1 # Publish a message to the global stream. nats --context east req js.in.global.orders 1 @@ -42,3 +55,7 @@ sleep 1 # Let's see the stream report again. nats --context east stream report + +nats --context east stream view ORDERS_EAST +nats --context central stream view ORDERS_CENTRAL +nats --context west stream view ORDERS_WEST From 81c4b1c634a9ce044f9ef42050d19a7476158161 Mon Sep 17 00:00:00 2001 From: Byron Ruth Date: Mon, 27 Mar 2023 21:39:26 -0400 Subject: [PATCH 3/3] Fix example Signed-off-by: Byron Ruth --- .../multi-region/cli/docker-compose.yaml | 36 +++++++++---------- examples/topologies/multi-region/cli/main.sh | 19 ++++------ 2 files changed, 24 insertions(+), 31 deletions(-) diff --git a/examples/topologies/multi-region/cli/docker-compose.yaml b/examples/topologies/multi-region/cli/docker-compose.yaml index 1460f157..bb417868 100644 --- a/examples/topologies/multi-region/cli/docker-compose.yaml +++ b/examples/topologies/multi-region/cli/docker-compose.yaml @@ -3,7 +3,7 @@ version: '3' services: n1.east.example.net: container_name: n1-east - image: synadia/nats-server:nightly-main + image: nats:2.9.15 dns_search: example.net environment: GATEWAY: c1 @@ -11,13 +11,13 @@ services: ADVERTISE: localhost:10000 REGION: region:east volumes: - - "./configs/cluster-east.conf:/nats/conf/nats-server.conf" + - "./configs/cluster-east.conf:/nats-server.conf" ports: - 10000:4222 - 10100:7422 n2.east.example.net: container_name: n2-east - image: synadia/nats-server:nightly-main + image: nats:2.9.15 dns_search: example.net labels: com.docker-tc.enabled: '1' @@ -27,13 +27,13 @@ services: ADVERTISE: localhost:10001 REGION: region:east volumes: - - "./configs/cluster-east.conf:/nats/conf/nats-server.conf" + - "./configs/cluster-east.conf:/nats-server.conf" ports: - 10001:4222 - 10101:7422 n3.east.example.net: container_name: n3-east - image: synadia/nats-server:nightly-main + image: nats:2.9.15 dns_search: example.net environment: GATEWAY: c1 @@ -41,13 +41,13 @@ services: ADVERTISE: localhost:10002 REGION: region:east volumes: - - "./configs/cluster-east.conf:/nats/conf/nats-server.conf" + - "./configs/cluster-east.conf:/nats-server.conf" ports: - 10002:4222 - 10102:7422 n1.central.example.net: container_name: n1-central - image: synadia/nats-server:nightly-main + image: nats:2.9.15 dns_search: example.net environment: GATEWAY: c1 @@ -55,13 +55,13 @@ services: ADVERTISE: localhost:10003 REGION: region:central volumes: - - "./configs/cluster-central.conf:/nats/conf/nats-server.conf" + - "./configs/cluster-central.conf:/nats-server.conf" ports: - 10003:4222 - 10103:7422 n2.central.example.net: container_name: n2-central - image: synadia/nats-server:nightly-main + image: nats:2.9.15 dns_search: example.net environment: GATEWAY: c1 @@ -69,13 +69,13 @@ services: ADVERTISE: localhost:10004 REGION: region:central volumes: - - "./configs/cluster-central.conf:/nats/conf/nats-server.conf" + - "./configs/cluster-central.conf:/nats-server.conf" ports: - 10004:4222 - 10104:7422 n3.central.example.net: container_name: n3-central - image: synadia/nats-server:nightly-main + image: nats:2.9.15 dns_search: example.net environment: GATEWAY: c1 @@ -83,13 +83,13 @@ services: ADVERTISE: localhost:10005 REGION: region:central volumes: - - "./configs/cluster-central.conf:/nats/conf/nats-server.conf" + - "./configs/cluster-central.conf:/nats-server.conf" ports: - 10005:4222 - 10105:7422 n1.west.example.net: container_name: n1-west - image: synadia/nats-server:nightly-main + image: nats:2.9.15 dns_search: example.net environment: GATEWAY: c1 @@ -97,13 +97,13 @@ services: ADVERTISE: localhost:10006 REGION: region:west volumes: - - "./configs/cluster-west.conf:/nats/conf/nats-server.conf" + - "./configs/cluster-west.conf:/nats-server.conf" ports: - 10006:4222 - 10106:7422 n2.west.example.net: container_name: n2-west - image: synadia/nats-server:nightly-main + image: nats:2.9.15 dns_search: example.net environment: GATEWAY: c1 @@ -111,13 +111,13 @@ services: ADVERTISE: localhost:10007 REGION: region:west volumes: - - "./configs/cluster-west.conf:/nats/conf/nats-server.conf" + - "./configs/cluster-west.conf:/nats-server.conf" ports: - 10007:4222 - 10107:7422 n3.west.example.net: container_name: n3-west - image: synadia/nats-server:nightly-main + image: nats:2.9.15 dns_search: example.net environment: GATEWAY: c1 @@ -125,7 +125,7 @@ services: ADVERTISE: localhost:10008 REGION: region:west volumes: - - "./configs/cluster-west.conf:/nats/conf/nats-server.conf" + - "./configs/cluster-west.conf:/nats-server.conf" ports: - 10008:4222 - 10108:7422 diff --git a/examples/topologies/multi-region/cli/main.sh b/examples/topologies/multi-region/cli/main.sh index 3560c716..676ab93b 100644 --- a/examples/topologies/multi-region/cli/main.sh +++ b/examples/topologies/multi-region/cli/main.sh @@ -28,14 +28,13 @@ nats context save central \ # Report the servers. nats --context east-sys server list -nats --context east stream add ORDERS_EAST --tag region:east --replicas 3 --subjects js.in.orders_east --defaults -nats --context west stream add ORDERS_WEST --tag region:west --replicas 3 --subjects js.in.orders_west --defaults -nats --context central stream add ORDERS_CENTRAL --tag region:central --replicas 3 --subjects js.in.orders_central --defaults - # Creating a region-local stream requires setting a tag for the desired region. -nats --context east stream edit ORDERS_EAST --force --config /app/ORDERS_EAST.json -nats --context west stream edit ORDERS_WEST --force --config /app/ORDERS_WEST.json -nats --context central stream edit ORDERS_CENTRAL --force --config /app/ORDERS_CENTRAL.json +nats --context east stream add --config /app/ORDERS_EAST.json +nats --context west stream add --config /app/ORDERS_WEST.json +nats --context central stream add --config /app/ORDERS_CENTRAL.json + +# Ensure the stream sourcing retries catch up. +sleep 5 # Creating a global stream involves ommitting the --tag option. nats --context east stream add --config /app/GLOBAL.json @@ -51,11 +50,5 @@ nats --context west req js.in.orders 1 # Publish a message to the global stream. nats --context east req js.in.global.orders 1 -sleep 1 - # Let's see the stream report again. nats --context east stream report - -nats --context east stream view ORDERS_EAST -nats --context central stream view ORDERS_CENTRAL -nats --context west stream view ORDERS_WEST