start docker as non-root

Author: joecorall

examples/ojs/main.tf

@@ -4,7 +4,7 @@ resource "random_shuffle" "zone" {
 }
 
 module "production" {
-  source = "git::https://github.com/libops/cloud-compose?ref=0.3.0"
+  source = "git::https://github.com/libops/cloud-compose?ref=0.4.0"
 
   name                = "ojs-production"
   project_id          = var.project_id
@@ -18,7 +18,7 @@ module "production" {
 }
 
 module "staging" {
-  source = "git::https://github.com/libops/cloud-compose?ref=0.3.0"
+  source = "git::https://github.com/libops/cloud-compose?ref=0.4.0"
 
   name                = "ojs-staging"
   project_id          = var.project_id

examples/ojs/terraform.tfvars

@@ -1,2 +1,4 @@
-docker_compose_repo   = "https://github.com/libops/ojs"
-docker_compose_init   = "docker compose up init"
+docker_compose_repo = "https://github.com/libops/ojs"
+docker_compose_init = [
+  "docker compose run init"
+]

main.tf

@@ -456,7 +456,7 @@ EOT
     project_id   = var.project_id
     zone         = var.zone
     name         = var.name
-    usePrivateIp = "true"
+    usePrivateIp = true
   }
   allowed_ips = tolist([
     "127.0.0.1/32",
@@ -490,7 +490,7 @@ module "ppb" {
   containers = tolist([
     {
       name   = "proxy-power-button",
-      image  = "us-docker.pkg.dev/libops-images/public/ppb:main",
+      image  = "us-docker.pkg.dev/libops-images/public/ppb:main@sha256:fc550f487fc8ab651dd5fa58399f862a58b706222aef75c3834ec5ff44f7ea1b",
       cpu    = "1000m"
       memory = "1Gi",
       port   = 8080

rootfs/etc/systemd/system/cloud-compose.service

@@ -1,9 +1,11 @@
 [Unit]
 Description=Docker Compose Application
-StartLimitIntervalSec=120
-StartLimitBurst=3
+PartOf=docker.service
+After=docker.service
 
 [Service]
+User=cloud-compose
+Group=cloud-compose
 EnvironmentFile=/home/cloud-compose/.env
 ExecStart=/mnt/disks/data/up
 ExecStop=/mnt/disks/data/down

rootfs/etc/systemd/system/internal-services.service

@@ -8,10 +8,10 @@ StartLimitIntervalSec=120
 StartLimitBurst=3
 
 [Service]
+User=cloud-compose
+Group=cloud-compose
 Environment="HOME=/home/cloud-compose"
 WorkingDirectory=/mnt/disks/data/libops-internal
-ExecStartPre=/bin/bash /home/cloud-compose/rotate-keys-internal.sh
-ExecStartPre=/usr/bin/test -f /mnt/disks/data/libops-internal/GOOGLE_APPLICATION_CREDENTIALS
 ExecStart=/usr/bin/docker compose up
 ExecStop=/usr/bin/docker compose down
 Restart=on-failure

rootfs/home/cloud-compose/app-init.sh

@@ -6,14 +6,19 @@ set -eou pipefail
 source /home/cloud-compose/profile.sh
 export HOME
 
+git config --global --add safe.directory "$DOCKER_COMPOSE_DIR"
+
 if [ ! -d "$DOCKER_COMPOSE_DIR" ]; then
-  mkdir -p "$DOCKER_COMPOSE_DIR"
   echo "Directory '$DOCKER_COMPOSE_DIR' not found. Cloning repository."
-  retry_until_success git clone -b "$DOCKER_COMPOSE_BRANCH" "$DOCKER_COMPOSE_REPO" "$DOCKER_COMPOSE_DIR"
+  mkdir -p "$DOCKER_COMPOSE_DIR"
+  pushd "$DOCKER_COMPOSE_DIR"
+  retry_until_success git clone -b "$DOCKER_COMPOSE_BRANCH" "$DOCKER_COMPOSE_REPO" .
+  chown -R cloud-compose:cloud-compose .
+else
+  pushd "$DOCKER_COMPOSE_DIR"
+  retry_until_success git pull origin "$DOCKER_COMPOSE_BRANCH"
 fi
 
-pushd "$DOCKER_COMPOSE_DIR"
-retry_until_success git pull origin "$DOCKER_COMPOSE_BRANCH"
 # set COMPOSE_PROJECT_NAME from value set in cloud-compose
 # sourced from /home/cloud-compose/profile.sh which loads /home/cloud-compose/.env
 update_env COMPOSE_PROJECT_NAME "$COMPOSE_PROJECT_NAME"

rootfs/home/cloud-compose/host-conf.sh

@@ -2,6 +2,9 @@
 
 set -eou pipefail
 
+# shellcheck disable=SC1091
+source /home/cloud-compose/profile.sh
+
 # block metadata server from docker and non-root
 /sbin/iptables -I FORWARD -d 169.254.169.254/32 -i docker0 -j DROP
 /sbin/iptables -A OUTPUT -m owner ! --uid-owner 0 -d 169.254.169.254/32 -p tcp --dport 80 -j DROP
@@ -11,28 +14,27 @@ systemctl restart fluent-bit
 systemctl restart docker
 
 # wait until our data-root /etc/docker/daemon.json setting are applied
-until test -d /mnt/disks/data/docker/volumes; do
-  echo "Waiting for docker volumes dir"
+until test -d /mnt/disks/data/docker/overlay2; do
+  echo "Waiting for docker overlay2 dir"
   sleep 1
 done
 
-# move volumes from docker's data root to our volumes disk
-rm -rf /mnt/disks/data/docker/volumes
-ln -s /mnt/disks/volumes /mnt/disks/data/docker/volumes
+if [ ! -d /home/cloud-compose/.docker/cli-plugins ]; then
+  mkdir -p /home/cloud-compose/.docker/cli-plugins
+fi
 
 # since COS is read only FS, install docker compose/buildx in home directory
 # and symlink to our data disk which can have executables
 if [ ! -f "/home/cloud-compose/.docker/cli-plugins/docker-compose" ]; then
-    curl -sSL \
+    retry_until_success curl -sSL \
         https://github.com/docker/compose/releases/download/v2.40.3/docker-compose-linux-x86_64 \
         -o /mnt/disks/data/docker-compose
     chmod o+x /mnt/disks/data/docker-compose
-    mkdir -p /home/cloud-compose/.docker/cli-plugins
     ln -sf /mnt/disks/data/docker-compose /home/cloud-compose/.docker/cli-plugins/docker-compose
 fi
 
 if [ ! -f "/home/cloud-compose/.docker/cli-plugins/docker-buildx" ]; then
-    curl -sSL \
+    retry_until_success curl -sSL \
         https://github.com/docker/buildx/releases/download/v0.30.1/buildx-v0.30.1.linux-amd64 \
         -o /mnt/disks/data/docker-buildx
     chmod o+x /mnt/disks/data/docker-buildx

rootfs/home/cloud-compose/host-init.sh

@@ -27,4 +27,8 @@ curl -sf \
 if ! diff <(md5sum .env.tmp) <(md5sum .env); then
   mv .env.tmp .env
   cp .env /mnt/disks/data/libops-internal/
+  chown cloud-compose /mnt/disks/data/libops-internal/.env
 fi
+
+chown -R cloud-compose:cloud-compose /home/cloud-compose
+usermod -aG docker cloud-compose

rootfs/home/cloud-compose/rotate-keys-app.sh

@@ -8,12 +8,14 @@ pushd /home/cloud-compose
 source /home/cloud-compose/profile.sh
 
 if [ ! -d "$DOCKER_COMPOSE_DIR/secrets" ]; then
-  mkdir "$DOCKER_COMPOSE_DIR/secrets"
+  mkdir -p "$DOCKER_COMPOSE_DIR/secrets"
 fi
 
 bash rotate-keys.sh \
     "$GCP_INSTANCE_NAME@$GCP_PROJECT.iam.gserviceaccount.com" \
     "$GCP_PROJECT" \
     "$DOCKER_COMPOSE_DIR/secrets/GOOGLE_APPLICATION_CREDENTIALS"
 
+chgrp cloud-compose "$DOCKER_COMPOSE_DIR/secrets/GOOGLE_APPLICATION_CREDENTIALS"
+
 popd

rootfs/home/cloud-compose/rotate-keys-internal.sh

@@ -12,4 +12,6 @@ bash rotate-keys.sh \
     "$GCP_PROJECT" \
     /mnt/disks/data/libops-internal/GOOGLE_APPLICATION_CREDENTIALS
 
+chgrp cloud-compose /mnt/disks/data/libops-internal/GOOGLE_APPLICATION_CREDENTIALS
+
 popd