KV on Kubernetes

Builds on Containers Runtime Configuration

Guide Not run when the site is built: it needs tools the build doesn't have Download project

On Compose (step 13) we named every node ourselves: kv@kv-a, kv@kv-b, and a fixed CLUSTER_NODES list. On Kubernetes that list doesn't exist. Pods come and go, get a new IP each time they start, and you scale them up and down with one number. So every node must discover the others at runtime, and each node must be named in a way the others can derive from what they discover.

The usual answer, and the one this step uses:

A headless Service (clusterIP: None): a DNS name whose lookup returns the IP of every KV pod.
Each pod names its node kv@<its pod IP>, from an environment variable Kubernetes fills in.
libcluster's Kubernetes.DNS strategy polls that DNS name and connects to kv@<ip> for every IP it gets back.
The cookie comes from a Kubernetes Secret, not from the image or the manifest.

The project

The application now takes its whole libcluster topology from configuration, so the strategy can differ between environments without code changes.

mix.exs

defmodule KV.MixProject do
  use Mix.Project

  def project do
    [app: :kv, version: "0.1.0", elixir: "~> 1.20", deps: deps()]
  end

  def application do
    [mod: {KV.Application, []}]
  end

  defp deps do
    [{:libcluster, "~> 3.4"}]
  end
end

mix.lock

%{
  "jason": {:hex, :jason, "1.4.5", "2e3a008590b0b8d7388c20293e9dcc9cf3e5d642fd2a114e4cbbb52e595d940a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "b0c823996102bcd0239b3c2444eb00409b72f6a140c1950bc8b457d836b30684"},
  "libcluster": {:hex, :libcluster, "3.6.0", "06085e9f67f78783608b81c89a62fa974d846f0b79569da4742441e57e4ca480", [:mix], [{:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.3", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "02b56995b57a38e12040d4ab0ce1bd20bece53d460970e665aa5c89b82a5b455"},
  "telemetry": {:hex, :telemetry, "1.4.2", "a0cb522801dffb1c49fe6e30561badffc7b6d0e180db1300df759faa22062855", [:rebar3], [], "hexpm", "928f6495066506077862c0d1646609eed891a4326bee3126ba54b60af61febb1"},
}

config/runtime.exs

import Config

# Every pod finds the others by asking DNS for the addresses behind the
# headless Service. Each answer is a pod IP, and the node there is called
# kv@<pod IP> (see RELEASE_NODE in k8s/deployment.yaml).
config :kv,
  topologies: [
    kv: [
      strategy: Cluster.Strategy.Kubernetes.DNS,
      config: [
        service: System.fetch_env!("CLUSTER_SERVICE"),
        application_name: "kv",
        polling_interval: 5_000
      ]
    ]
  ]

lib/kv/application.ex

defmodule KV.Application do
  use Application

  @impl true
  def start(_type, _args) do
    topologies = Application.fetch_env!(:kv, :topologies)

    children = [
      %{id: :pg, start: {:pg, :start_link, []}},
      {Cluster.Supervisor, [topologies, [name: KV.ClusterSupervisor]]},
      KV.Replica
    ]

    Supervisor.start_link(children, strategy: :one_for_one, name: KV.Supervisor)
  end
end

lib/kv/replica.ex

defmodule KV.Replica do
  @moduledoc "One replica of the store; every write goes to all of them (step 8)."
  use GenServer

  def start_link(_opts), do: GenServer.start_link(__MODULE__, %{}, name: __MODULE__)

  def put(key, value) do
    for replica <- :pg.get_members(:kv_replicas) do
      :ok = GenServer.call(replica, {:put, key, value})
    end

    :ok
  end

  def get(key), do: GenServer.call(__MODULE__, {:get, key})

  @doc "How many replicas this node can see."
  def count, do: length(:pg.get_members(:kv_replicas))

  @impl true
  def init(map) do
    :ok = :pg.join(:kv_replicas, self())
    {:ok, map}
  end

  @impl true
  def handle_call({:put, k, v}, _from, map), do: {:reply, :ok, Map.put(map, k, v)}
  def handle_call({:get, k}, _from, map), do: {:reply, Map.get(map, k), map}
end

The image is built exactly as in step 13.

Dockerfile

# Same two-stage layout as on the Containers page.
ARG ELIXIR_VERSION=1.20.5
ARG OTP_VERSION=29.1.2
ARG DEBIAN_VERSION=trixie-20261104-slim

ARG BUILDER_IMAGE="hexpm/elixir:${ELIXIR_VERSION}-erlang-${OTP_VERSION}-debian-${DEBIAN_VERSION}"
ARG RUNNER_IMAGE="debian:${DEBIAN_VERSION}"

FROM ${BUILDER_IMAGE} AS builder

WORKDIR /app
RUN mix local.hex --force && mix local.rebar --force
ENV MIX_ENV="prod"

# The lock file pins the exact dependency versions that were tested.
COPY mix.exs mix.lock ./
RUN mix deps.get --only $MIX_ENV && mix deps.compile

COPY lib lib
# runtime.exs is read at boot, so changing it doesn't recompile anything.
COPY config/runtime.exs config/
RUN mix release

FROM ${RUNNER_IMAGE} AS final

RUN apt-get update \
  && apt-get install -y --no-install-recommends libstdc++6 openssl libncurses6 locales ca-certificates \
  && rm -rf /var/lib/apt/lists/*

RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen
ENV LANG=en_US.UTF-8
ENV LANGUAGE=en_US:en
ENV LC_ALL=en_US.UTF-8

WORKDIR /app
RUN chown nobody /app
COPY --from=builder --chown=nobody:root /app/_build/prod/rel/kv ./

USER nobody
CMD ["/app/bin/kv", "start"]

.dockerignore

1
2
3

_build
deps
.git

The Kubernetes objects

k8s/service.yaml

# A headless Service (clusterIP: None) has no virtual IP: a DNS lookup of its
# name returns the IP of every matching pod. That list is what libcluster
# polls.
apiVersion: v1
kind: Service
metadata:
  name: kv-headless
spec:
  clusterIP: None
  selector:
    app: kv
  # List pods before they are "ready", so nodes can cluster while starting.
  publishNotReadyAddresses: true
  ports:
    - name: epmd
      port: 4369

Kubernetes expands $(POD_IP) in RELEASE_NODE from the variable defined before it, so each pod gets a unique node name that matches what libcluster will build from DNS.

k8s/deployment.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: kv
spec:
  replicas: 3
  selector:
    matchLabels:
      app: kv
  template:
    metadata:
      labels:
        app: kv
    spec:
      # Time between SIGTERM and SIGKILL: enough for a clean shutdown.
      terminationGracePeriodSeconds: 30
      containers:
        - name: kv
          image: kv:0.1.0
          env:
            # The pod's own IP, filled in by Kubernetes...
            - name: POD_IP
              valueFrom:
                fieldRef:
                  fieldPath: status.podIP
            # ...becomes the host part of the node name: kv@10.1.2.3.
            - name: RELEASE_DISTRIBUTION
              value: name
            - name: RELEASE_NODE
              value: kv@$(POD_IP)
            - name: RELEASE_COOKIE
              valueFrom:
                secretKeyRef:
                  name: kv-cookie
                  key: cookie
            - name: CLUSTER_SERVICE
              value: kv-headless

Deploy

Build the image and make it available to your cluster: push it to a registry the cluster pulls from, or load it into a local cluster (kind load docker-image kv:0.1.0 for kind). Then create the cookie as a Secret and apply the manifests.

$ docker build -t kv:0.1.0 .
$ kubectl create secret generic kv-cookie --from-literal=cookie="$(openssl rand -base64 32)"
$ kubectl apply -f k8s/
$ kubectl rollout status deployment/kv

Every pod should see the other two. kubectl exec runs the release script in a pod, with that pod's environment, so rpc reaches the node running there.

$ PODS=$(kubectl get pods -l app=kv -o jsonpath='{.items[*].metadata.name}')
$ set -- $PODS
$ kubectl exec "$1" -- bin/kv rpc 'IO.inspect(length(Node.list()), label: "connected to")'
$ kubectl exec "$1" -- bin/kv rpc 'KV.Replica.put("lang", "elixir")'
$ kubectl exec "$2" -- bin/kv rpc 'IO.inspect(KV.Replica.get("lang"))'

Scale up, and the new pod joins by itself within a polling interval:

$ kubectl scale deployment/kv --replicas=4

Pitfall: a new pod starts with an empty store, and a rolling update replaces every pod. With this KV (in-memory replicas, as in step 8) a rollout can lose all data. Data that must survive belongs in a database, or in Mnesia with disc_copies on a StatefulSet with persistent volumes.

Pitfall: pod IPs change on every restart, so node names change too. Anything that remembers node names (Mnesia's schema, for one) needs stable names: run a StatefulSet with a headless Service and use the Kubernetes.DNSSRV strategy, which names nodes after the pods' stable DNS names instead of their IPs.

Pitfall: publishNotReadyAddresses: true lets nodes find each other before they pass their readiness probe. Without it, a pod that waits for the cluster before becoming ready, and a cluster that waits for ready pods, wait for each other forever.

Pitfall: Erlang distribution gives anyone holding the cookie full control of every node. Keep the cookie in a Secret, and use a NetworkPolicy so only KV pods can reach ports 4369 and the distribution port.

Improve this page on GitHub