From 19319dba9cff383b3107b2f0c8df327a903a7282 Mon Sep 17 00:00:00 2001 From: Trevor Vallender Date: Fri, 29 Sep 2023 10:20:41 +0100 Subject: [PATCH] Setting up deployment with Kamal --- .gitattributes | 2 + .gitignore | 2 + .kamal/hooks/post-deploy.sample | 14 ++++ .kamal/hooks/pre-build.sample | 51 +++++++++++++++ .kamal/hooks/pre-connect.sample | 47 ++++++++++++++ .kamal/hooks/pre-deploy.sample | 109 ++++++++++++++++++++++++++++++++ Dockerfile | 57 +++++++++++++++++ Gemfile | 1 + Gemfile.lock | 23 +++++++ bin/docker-entrypoint | 9 +++ config/credentials.yml.enc | 2 +- config/database.yml | 3 +- config/deploy.yml | 55 ++++++++++++++++ 13 files changed, 373 insertions(+), 2 deletions(-) create mode 100755 .kamal/hooks/post-deploy.sample create mode 100755 .kamal/hooks/pre-build.sample create mode 100755 .kamal/hooks/pre-connect.sample create mode 100755 .kamal/hooks/pre-deploy.sample create mode 100644 Dockerfile create mode 100755 bin/docker-entrypoint create mode 100644 config/deploy.yml diff --git a/.gitattributes b/.gitattributes index 31eeee0..8dc4323 100644 --- a/.gitattributes +++ b/.gitattributes @@ -5,3 +5,5 @@ db/schema.rb linguist-generated # Mark any vendored files as having been vendored. vendor/* linguist-vendored +config/credentials/*.yml.enc diff=rails_credentials +config/credentials.yml.enc diff=rails_credentials diff --git a/.gitignore b/.gitignore index 73c5d10..88389d2 100644 --- a/.gitignore +++ b/.gitignore @@ -40,3 +40,5 @@ devenv.local.nix # pre-commit .pre-commit-config.yaml +# Kamal +.env diff --git a/.kamal/hooks/post-deploy.sample b/.kamal/hooks/post-deploy.sample new file mode 100755 index 0000000..75efafc --- /dev/null +++ b/.kamal/hooks/post-deploy.sample @@ -0,0 +1,14 @@ +#!/bin/sh + +# A sample post-deploy hook +# +# These environment variables are available: +# KAMAL_RECORDED_AT +# KAMAL_PERFORMER +# KAMAL_VERSION +# KAMAL_HOSTS +# KAMAL_ROLE (if set) +# KAMAL_DESTINATION (if set) +# KAMAL_RUNTIME + +echo "$KAMAL_PERFORMER deployed $KAMAL_VERSION to $KAMAL_DESTINATION in $KAMAL_RUNTIME seconds" diff --git a/.kamal/hooks/pre-build.sample b/.kamal/hooks/pre-build.sample new file mode 100755 index 0000000..218e296 --- /dev/null +++ b/.kamal/hooks/pre-build.sample @@ -0,0 +1,51 @@ +#!/bin/sh + +# A sample pre-build hook +# +# Checks: +# 1. We have a clean checkout +# 2. A remote is configured +# 3. The branch has been pushed to the remote +# 4. The version we are deploying matches the remote +# +# These environment variables are available: +# KAMAL_RECORDED_AT +# KAMAL_PERFORMER +# KAMAL_VERSION +# KAMAL_HOSTS +# KAMAL_ROLE (if set) +# KAMAL_DESTINATION (if set) + +if [ -n "$(git status --porcelain)" ]; then + echo "Git checkout is not clean, aborting..." >&2 + git status --porcelain >&2 + exit 1 +fi + +first_remote=$(git remote) + +if [ -z "$first_remote" ]; then + echo "No git remote set, aborting..." >&2 + exit 1 +fi + +current_branch=$(git branch --show-current) + +if [ -z "$current_branch" ]; then + echo "No git remote set, aborting..." >&2 + exit 1 +fi + +remote_head=$(git ls-remote $first_remote --tags $current_branch | cut -f1) + +if [ -z "$remote_head" ]; then + echo "Branch not pushed to remote, aborting..." >&2 + exit 1 +fi + +if [ "$KAMAL_VERSION" != "$remote_head" ]; then + echo "Version ($KAMAL_VERSION) does not match remote HEAD ($remote_head), aborting..." >&2 + exit 1 +fi + +exit 0 diff --git a/.kamal/hooks/pre-connect.sample b/.kamal/hooks/pre-connect.sample new file mode 100755 index 0000000..18e61d7 --- /dev/null +++ b/.kamal/hooks/pre-connect.sample @@ -0,0 +1,47 @@ +#!/usr/bin/env ruby + +# A sample pre-connect check +# +# Warms DNS before connecting to hosts in parallel +# +# These environment variables are available: +# KAMAL_RECORDED_AT +# KAMAL_PERFORMER +# KAMAL_VERSION +# KAMAL_HOSTS +# KAMAL_ROLE (if set) +# KAMAL_DESTINATION (if set) +# KAMAL_RUNTIME + +hosts = ENV["KAMAL_HOSTS"].split(",") +results = nil +max = 3 + +elapsed = Benchmark.realtime do + results = hosts.map do |host| + Thread.new do + tries = 1 + + begin + Socket.getaddrinfo(host, 0, Socket::AF_UNSPEC, Socket::SOCK_STREAM, nil, Socket::AI_CANONNAME) + rescue SocketError + if tries < max + puts "Retrying DNS warmup: #{host}" + tries += 1 + sleep rand + retry + else + puts "DNS warmup failed: #{host}" + host + end + end + + tries + end + end.map(&:value) +end + +retries = results.sum - hosts.size +nopes = results.count { |r| r == max } + +puts "Prewarmed %d DNS lookups in %.2f sec: %d retries, %d failures" % [ hosts.size, elapsed, retries, nopes ] diff --git a/.kamal/hooks/pre-deploy.sample b/.kamal/hooks/pre-deploy.sample new file mode 100755 index 0000000..1b280c7 --- /dev/null +++ b/.kamal/hooks/pre-deploy.sample @@ -0,0 +1,109 @@ +#!/usr/bin/env ruby + +# A sample pre-deploy hook +# +# Checks the Github status of the build, waiting for a pending build to complete for up to 720 seconds. +# +# Fails unless the combined status is "success" +# +# These environment variables are available: +# KAMAL_RECORDED_AT +# KAMAL_PERFORMER +# KAMAL_VERSION +# KAMAL_HOSTS +# KAMAL_COMMAND +# KAMAL_SUBCOMMAND +# KAMAL_ROLE (if set) +# KAMAL_DESTINATION (if set) + +# Only check the build status for production deployments +if ENV["KAMAL_COMMAND"] == "rollback" || ENV["KAMAL_DESTINATION"] != "production" + exit 0 +end + +require "bundler/inline" + +# true = install gems so this is fast on repeat invocations +gemfile(true, quiet: true) do + source "https://rubygems.org" + + gem "octokit" + gem "faraday-retry" +end + +MAX_ATTEMPTS = 72 +ATTEMPTS_GAP = 10 + +def exit_with_error(message) + $stderr.puts message + exit 1 +end + +class GithubStatusChecks + attr_reader :remote_url, :git_sha, :github_client, :combined_status + + def initialize + @remote_url = `git config --get remote.origin.url`.strip.delete_prefix("https://github.com/") + @git_sha = `git rev-parse HEAD`.strip + @github_client = Octokit::Client.new(access_token: ENV["GITHUB_TOKEN"]) + refresh! + end + + def refresh! + @combined_status = github_client.combined_status(remote_url, git_sha) + end + + def state + combined_status[:state] + end + + def first_status_url + first_status = combined_status[:statuses].find { |status| status[:state] == state } + first_status && first_status[:target_url] + end + + def complete_count + combined_status[:statuses].count { |status| status[:state] != "pending"} + end + + def total_count + combined_status[:statuses].count + end + + def current_status + if total_count > 0 + "Completed #{complete_count}/#{total_count} checks, see #{first_status_url} ..." + else + "Build not started..." + end + end +end + + +$stdout.sync = true + +puts "Checking build status..." +attempts = 0 +checks = GithubStatusChecks.new + +begin + loop do + case checks.state + when "success" + puts "Checks passed, see #{checks.first_status_url}" + exit 0 + when "failure" + exit_with_error "Checks failed, see #{checks.first_status_url}" + when "pending" + attempts += 1 + end + + exit_with_error "Checks are still pending, gave up after #{MAX_ATTEMPTS * ATTEMPTS_GAP} seconds" if attempts == MAX_ATTEMPTS + + puts checks.current_status + sleep(ATTEMPTS_GAP) + checks.refresh! + end +rescue Octokit::NotFound + exit_with_error "Build status could not be found" +end diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..96161ae --- /dev/null +++ b/Dockerfile @@ -0,0 +1,57 @@ +ARG RUBY_VERSION=3.2.2 +FROM registry.docker.com/library/ruby:$RUBY_VERSION-slim as base + +# Rails app lives here +WORKDIR /rails + +# Set production environment +ENV RAILS_ENV="production" \ + BUNDLE_DEPLOYMENT="1" \ + BUNDLE_PATH="/usr/local/bundle" \ + BUNDLE_WITHOUT="development" + +# Throw-away build stage to reduce size of final image +FROM base as build + +# Install packages needed to build gems +RUN apt-get update -qq && \ + apt-get install --no-install-recommends -y build-essential curl git libpq-dev libvips pkg-config + +# Install application gems +COPY Gemfile Gemfile.lock ./ +RUN bundle install && \ + rm -rf ~/.bundle/ "${BUNDLE_PATH}"/ruby/*/cache "${BUNDLE_PATH}"/ruby/*/bundler/gems/*/.git && \ + bundle exec bootsnap precompile --gemfile + +# Copy application code +COPY . . + +# Precompile bootsnap code for faster boot times +RUN bundle exec bootsnap precompile app/ lib/ + +# Precompiling assets for production without requiring secret RAILS_MASTER_KEY +RUN SECRET_KEY_BASE_DUMMY=1 ./bin/rails assets:precompile + +# Final stage for app image +FROM base + +# Install packages needed for deployment +RUN apt-get update -qq && \ + apt-get install --no-install-recommends -y libvips postgresql-client curl && \ + rm -rf /var/lib/apt/lists /var/cache/apt/archives + +# Copy built artifacts: gems, application +COPY --from=build /usr/local/bundle /usr/local/bundle +COPY --from=build /rails /rails + +# Run and own only the runtime files as a non-root user for security +RUN useradd rails --home /rails --shell /bin/bash && \ + chown -R rails:rails db log storage tmp +USER rails:rails + +# Entrypoint prepares the database. +ENTRYPOINT ["/rails/bin/docker-entrypoint"] + +# Start the server by default, this can be overwritten at runtime +EXPOSE 3000 +CMD ["./bin/rails", "server"] diff --git a/Gemfile b/Gemfile index b406b30..44483a5 100644 --- a/Gemfile +++ b/Gemfile @@ -19,6 +19,7 @@ group :development, :test do end group :development do + gem "kamal" gem "web-console" end diff --git a/Gemfile.lock b/Gemfile.lock index 25dfe14..81a6ad4 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -69,6 +69,7 @@ GEM addressable (2.8.5) public_suffix (>= 2.0.2, < 6.0) bcrypt (3.1.19) + bcrypt_pbkdf (1.1.0) bindex (0.8.1) bootsnap (1.16.0) msgpack (~> 1.2) @@ -88,6 +89,8 @@ GEM debug (1.8.0) irb (>= 1.5.0) reline (>= 0.3.1) + dotenv (2.8.1) + ed25519 (1.3.0) erubi (1.12.0) globalid (1.1.0) activesupport (>= 5.0) @@ -102,6 +105,16 @@ GEM jbuilder (2.11.5) actionview (>= 5.0.0) activesupport (>= 5.0.0) + kamal (1.0.0) + activesupport (>= 7.0) + bcrypt_pbkdf (~> 1.0) + concurrent-ruby (~> 1.2) + dotenv (~> 2.8) + ed25519 (~> 1.2) + net-ssh (~> 7.0) + sshkit (~> 1.21) + thor (~> 1.2) + zeitwerk (~> 2.5) loofah (2.21.3) crass (~> 1.0.2) nokogiri (>= 1.12.0) @@ -123,9 +136,14 @@ GEM net-protocol net-protocol (0.2.1) timeout + net-scp (4.0.0) + net-ssh (>= 2.6.5, < 8.0.0) net-smtp (0.3.3) net-protocol + net-ssh (7.2.0) nio4r (2.5.9) + nokogiri (1.15.3-aarch64-linux) + racc (~> 1.4) nokogiri (1.15.3-x86_64-linux) racc (~> 1.4) pg (1.5.3) @@ -181,6 +199,9 @@ GEM actionpack (>= 5.2) activesupport (>= 5.2) sprockets (>= 3.0.0) + sshkit (1.21.5) + net-scp (>= 1.1.2) + net-ssh (>= 2.8.0) stimulus-rails (1.2.2) railties (>= 6.0.0) thor (1.2.2) @@ -205,6 +226,7 @@ GEM zeitwerk (2.6.11) PLATFORMS + aarch64-linux x86_64-linux DEPENDENCIES @@ -214,6 +236,7 @@ DEPENDENCIES debug importmap-rails jbuilder + kamal pg puma rails (~> 7.0) diff --git a/bin/docker-entrypoint b/bin/docker-entrypoint new file mode 100755 index 0000000..7a82653 --- /dev/null +++ b/bin/docker-entrypoint @@ -0,0 +1,9 @@ +#!/bin/bash -e + +# If running the rails server then create or migrate existing database +if [ "${*}" == "./bin/rails server -p 3000" ]; then + ./bin/rails db:prepare +fi + +exec "${@}" + diff --git a/config/credentials.yml.enc b/config/credentials.yml.enc index 2abd854..ba2a3d5 100644 --- a/config/credentials.yml.enc +++ b/config/credentials.yml.enc @@ -1 +1 @@ -fHW+PxXiyPTwXEz1Z7uhfhJwqM6JBznqReo8JCAtnP0qYFm1ohkBnotFVPG3e62aDIIkv75AzkDd7PMNz6mNSB5k33hcUelsJQu1NfzhlcTbU+3DjJidCyrBA/Q/gHU5751Wy5ia2plmefULqGJpk8BEJxw5T08MSWw5ckMSd6HW+GsV7Um/uTFIAIB1OTRpXezTGyPKycbv6z5+EbzP2WGX1xYNWYLwqHk6yLP1lv2DL8o7SvX0M39mwSfkhCa3LDDB1MB0lgxYnXahP0IPfCg21w2VjS5dwrTyq1T6KGz6c/Fma+jTJ4FPyzjyGEIxEcMkNLALlefgC4y+QjniMFuEHmXnTeMJ1AkGA4S+jNV4xdatKA4BkMV3QbdOSsjhZdTjZP5fPUd3rG9Xxi/FRLgmtJqM1zKbmwCY--2RJuax7ovSghbzYP--UxUYrxSJUDF72/VwwkeTOQ== \ No newline at end of file +lAYjAMyj72J5LU+gQCHjaDBVeL4rGCpP9JU2lzRCEonWIJ0c2bdo+Gn8RkMdJBScwrc8tBU72gv63mLn8mbFrezDmVvsAXLgpYbo6Pp5y+8VHkD+VZqgKZhzAr27i625X9Xmgsc7ik5A/B9Tk9jsyR5lXLNqZC99eKwG86nE2uUlAIN1vGJDTqIwVowQm1JRLANywaFZ+sxvBFzmwGlcPcOTpCobtIVa7ai3zMuXEWuDF+VChIGhzB1RJTW+hSm8oUaGwsE8T5lhHw8m2OJqn342L8OOsDVLN1o3Us/F/2fDo2v1pRNNLRbfSvVPZwKTVjtxjLuP0c+8B6e4M9qanB8ke681wS7ksJWrRPT4ny0DJH1GVb3Pd8rCWIkKMSiYhz2A4yVCOXA9ctfXkWH8RLteSa2I--KBW8eCVEbtvoEhdb--/feRW0xcGPEchSxObA/6WQ== \ No newline at end of file diff --git a/config/database.yml b/config/database.yml index 0b2538a..05cea76 100644 --- a/config/database.yml +++ b/config/database.yml @@ -18,4 +18,5 @@ test: production: <<: *default database: soc_production - password: <%= ENV["SOC_DATABASE_PASSWORD"] %> + password: <%= ENV["POSTGRES_PASSWORD"] %> + host: <%= ENV.fetch("DB_HOST") { 'localhost' } %> diff --git a/config/deploy.yml b/config/deploy.yml new file mode 100644 index 0000000..a703155 --- /dev/null +++ b/config/deploy.yml @@ -0,0 +1,55 @@ +service: soc + +image: tsvallender/soc + +traefik: + host_port: 8000 + +servers: + - tsvallender.co.uk + +registry: + username: tsvallender + password: + - KAMAL_REGISTRY_PASSWORD + +env: + clear: + APPLICATION_HOST: tsvallender.co.uk + DB_HOST: 172.17.0.2 + RAILS_LOG_TO_STDOUT: true + RAILS_SERVE_STATIC_FILES: true + secret: + - RAILS_MASTER_KEY + - POSTGRES_PASSWORD + +ssh: + user: kamal + +builder: + args: + APPLICATION_HOST: tsvallender.co.uk + RUBY_VERSION: 3.2.2 + +accessories: + postgres: + image: postgres:16 + host: tsvallender.co.uk + env: + clear: + POSTGRES_USER: soc + secret: + - POSTGRES_PASSWORD + volumes: + - /srv/soc/data:/var/lib/postgresql/data + options: + ip: 172.17.0.2 + +builder: + remote: + arch: arm64 + host: ssh://kamal@tsvallender.co.uk + +healthcheck: + path: /sessions/new + port: 3000