KV in containers

Guide Not run when the site is built: it needs tools the build doesn't have Download project

Step 12 started two KV nodes from one release, on one machine, configured only by environment variables. This step does the same with containers: one image, two containers, each told who it is by its environment. It's the shape KV will have on Kubernetes in the next step, small enough to run on a laptop with Docker Compose.

On this page:

KV from step 12, plus a mix.lock that pins its dependencies.
A Dockerfile built like the one on the Containers page.
A compose.yaml that starts two containers from that image and gives each a node name.
Start the cluster, write on one node, read on the other.

The project

Unchanged from step 12, except for the lock file. mix deps.get writes it the first time; commit it, so every image is built with exactly the versions you tested.

mix.exs

defmodule KV.MixProject do
  use Mix.Project

  def project do
    [app: :kv, version: "0.1.0", elixir: "~> 1.20", deps: deps()]
  end

  def application do
    [mod: {KV.Application, []}]
  end

  defp deps do
    [{:libcluster, "~> 3.4"}]
  end
end

mix.lock

%{
  "jason": {:hex, :jason, "1.4.5", "2e3a008590b0b8d7388c20293e9dcc9cf3e5d642fd2a114e4cbbb52e595d940a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "b0c823996102bcd0239b3c2444eb00409b72f6a140c1950bc8b457d836b30684"},
  "libcluster": {:hex, :libcluster, "3.6.0", "06085e9f67f78783608b81c89a62fa974d846f0b79569da4742441e57e4ca480", [:mix], [{:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.3", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "02b56995b57a38e12040d4ab0ce1bd20bece53d460970e665aa5c89b82a5b455"},
  "telemetry": {:hex, :telemetry, "1.4.2", "a0cb522801dffb1c49fe6e30561badffc7b6d0e180db1300df759faa22062855", [:rebar3], [], "hexpm", "928f6495066506077862c0d1646609eed891a4326bee3126ba54b60af61febb1"},
}

config/runtime.exs

import Config

# The nodes to cluster with, read when each node boots:
# CLUSTER_NODES=kv_a@127.0.0.1,kv_b@127.0.0.1
nodes =
  System.fetch_env!("CLUSTER_NODES")
  |> String.split(",", trim: true)
  |> Enum.map(&String.to_atom/1)

config :kv, cluster_nodes: nodes

lib/kv/application.ex

defmodule KV.Application do
  use Application

  @impl true
  def start(_type, _args) do
    hosts = Application.fetch_env!(:kv, :cluster_nodes)
    topologies = [kv: [strategy: Cluster.Strategy.Epmd, config: [hosts: hosts, timeout: 500]]]

    children = [
      %{id: :pg, start: {:pg, :start_link, []}},
      {Cluster.Supervisor, [topologies, [name: KV.ClusterSupervisor]]},
      KV.Replica
    ]

    Supervisor.start_link(children, strategy: :one_for_one, name: KV.Supervisor)
  end
end

lib/kv/replica.ex

defmodule KV.Replica do
  @moduledoc "One replica of the store; every write goes to all of them (step 8)."
  use GenServer

  def start_link(_opts), do: GenServer.start_link(__MODULE__, %{}, name: __MODULE__)

  def put(key, value) do
    for replica <- :pg.get_members(:kv_replicas) do
      :ok = GenServer.call(replica, {:put, key, value})
    end

    :ok
  end

  def get(key), do: GenServer.call(__MODULE__, {:get, key})

  @doc "How many replicas this node can see."
  def count, do: length(:pg.get_members(:kv_replicas))

  @impl true
  def init(map) do
    :ok = :pg.join(:kv_replicas, self())
    {:ok, map}
  end

  @impl true
  def handle_call({:put, k, v}, _from, map), do: {:reply, :ok, Map.put(map, k, v)}
  def handle_call({:get, k}, _from, map), do: {:reply, Map.get(map, k), map}
end

The image

Dockerfile

# Same two-stage layout as on the Containers page.
ARG ELIXIR_VERSION=1.20.5
ARG OTP_VERSION=29.1.2
ARG DEBIAN_VERSION=trixie-20261104-slim

ARG BUILDER_IMAGE="hexpm/elixir:${ELIXIR_VERSION}-erlang-${OTP_VERSION}-debian-${DEBIAN_VERSION}"
ARG RUNNER_IMAGE="debian:${DEBIAN_VERSION}"

FROM ${BUILDER_IMAGE} AS builder

WORKDIR /app
RUN mix local.hex --force && mix local.rebar --force
ENV MIX_ENV="prod"

# The lock file pins the exact dependency versions that were tested.
COPY mix.exs mix.lock ./
RUN mix deps.get --only $MIX_ENV && mix deps.compile

COPY lib lib
# runtime.exs is read at boot, so changing it doesn't recompile anything.
COPY config/runtime.exs config/
RUN mix release

FROM ${RUNNER_IMAGE} AS final

RUN apt-get update \
  && apt-get install -y --no-install-recommends libstdc++6 openssl libncurses6 locales ca-certificates \
  && rm -rf /var/lib/apt/lists/*

RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen
ENV LANG=en_US.UTF-8
ENV LANGUAGE=en_US:en
ENV LC_ALL=en_US.UTF-8

WORKDIR /app
RUN chown nobody /app
COPY --from=builder --chown=nobody:root /app/_build/prod/rel/kv ./

USER nobody
CMD ["/app/bin/kv", "start"]

.dockerignore

1
2
3

_build
deps
.git

Two nodes, one image

A node name has two parts, name@host. With RELEASE_DISTRIBUTION=sname the host part is the machine's short hostname, here the container's hostname. Compose names each container's network address after its service, so kv-a and kv-b resolve inside the network and kv@kv-a can reach kv@kv-b.

compose.yaml

# Two KV nodes from one image. Each container's hostname is the host part of
# its node name (kv@kv-a, kv@kv-b), and Compose's network resolves those
# names, so the nodes can reach each other.
x-kv-env: &kv-env
  RELEASE_DISTRIBUTION: sname
  RELEASE_NODE: kv
  # Demo only: real deployments read the cookie from a secret.
  RELEASE_COOKIE: kv-demo-cookie
  CLUSTER_NODES: kv@kv-a,kv@kv-b

services:
  kv-a:
    build: .
    hostname: kv-a
    environment: *kv-env
  kv-b:
    build: .
    hostname: kv-b
    environment: *kv-env

Run it

$ docker compose up -d --build

Wait until node kv-a sees both replicas, then write there and read on kv-b. docker compose exec runs the release script inside a container, with that container's environment, so rpc talks to the node running there.

$ until docker compose exec kv-a bin/kv rpc 'IO.write(KV.Replica.count())' 2>/dev/null | grep -q 2; do
  sleep 0.5
done
$ docker compose exec kv-a bin/kv rpc 'KV.Replica.put("lang", "elixir")'
$ docker compose exec kv-b bin/kv rpc 'IO.inspect(KV.Replica.get("lang"))'

$ docker compose down

Pitfall: Erlang distribution needs more than one port between nodes: EPMD on 4369 and a port for each node. Inside one Compose network every port is open; across hosts or firewalls you must open them, or pin the distribution port range (inet_dist_listen_min/inet_dist_listen_max in rel/vm.args.eex).

Pitfall: the cookie in compose.yaml is fine for a demo and nothing else. Anyone with the cookie who can reach the ports can run any code on your nodes. Keep it in your platform's secret store.

Improve this page on GitHub