0

We run many services under runit and it has been great.

Recently started using Sidekiq which is amazing. It's running it under runit. The issue is that when running "sv stop ." it's not stopping the process. Calling restart actually brings up another instance of sidekiq leaving the old one running.

Here's our sample run file:

#!/bin/sh
cd /PATH_TO_SIDEKIQ
exec 2>&1
export DB_POOL_SIZE=25
exec bundle exec sidekiq -e production 2>&1 | logger -p local2.info -t sidekiq

The pid file in supervise/pid matches the running pid.

Thanks, jeremy

tippingpints
  • 1
  • 2

2 Answers2

0

OK I'm not sure the exact cause but I think it's how SideKiq is started. It get's it's own pid and runit assigns one as well. Anyways I've solved this by modifying the built in capistrano recipe.

Basic idea is to use the sidekiqctl to send the :quite signal so workers stop accepting new work. Then on :stop we stop runit then send the sidekiqctl :stop signal. On start we use runit to start the process again.

Capistrano::Configuration.instance.load do

  _cset(:sidekiq_default_hooks) { true }
  _cset(:sidekiq_cmd) { "#{fetch(:bundle_cmd, "bundle")} exec sidekiq" }
  _cset(:sidekiqctl_cmd) { "#{fetch(:bundle_cmd, "bundle")} exec sidekiqctl" }
  _cset(:sidekiq_timeout)   { 10 }
  _cset(:sidekiq_role)      { :app }
  _cset(:sidekiq_pid)       { "#{current_path}/tmp/pids/sidekiq.pid" }
  _cset(:sidekiq_processes) { 1 }

  if fetch(:sidekiq_default_hooks)
    before "deploy:update_code", "sidekiq:quiet"
    after "deploy:stop",    "sidekiq:stop"
    after "deploy:start",   "sidekiq:start"
    before "deploy:restart", "sidekiq:restart"
  end

  namespace :sidekiq do
    def for_each_process(&block)
      fetch(:sidekiq_processes).times do |idx|
        yield((idx == 0 ? "#{fetch(:sidekiq_pid)}" : "#{fetch(:sidekiq_pid)}-#{idx}"), idx)
      end
    end

    desc "Quiet sidekiq (stop accepting new work)"
    task :quiet, :roles => lambda { fetch(:sidekiq_role) }, :on_no_matching_servers => :continue do
      for_each_process do |pid_file, idx|
        run "if [ -d #{current_path} ] && [ -f #{pid_file} ] && kill -0 `cat #{pid_file}`> /dev/null 2>&1; then cd #{current_path} && #{fetch(:sidekiqctl_cmd)} quiet #{pid_file} ; else echo 'Sidekiq is not running'; fi"
      end
    end

    desc "Stop sidekiq"
    task :stop, :roles => lambda { fetch(:sidekiq_role) }, :on_no_matching_servers => :continue do
      for_each_process do |pid_file, idx|
        sidekiq_dir = idx == 0 ? "/SERVICE_DIR/sidekiq" : "/SERVICE_DIR/sidekiq-#{idx}"
        run "sv stop #{sidekiq_dir}"
        run "if [ -d #{current_path} ] && [ -f #{pid_file} ] && kill -0 `cat #{pid_file}`> /dev/null 2>&1; then cd #{current_path} && #{fetch(:sidekiqctl_cmd)} stop #{pid_file} #{fetch :sidekiq_timeout} ; else echo 'Sidekiq is not running'; fi"
      end
    end

    desc "Start sidekiq"
    task :start, :roles => lambda { fetch(:sidekiq_role) }, :on_no_matching_servers => :continue do
      rails_env = fetch(:rails_env, "production")
      for_each_process do |pid_file, idx|
        sidekiq_dir = idx == 0 ? "/SERVICE_DIR/sidekiq" : "/SERVICE_DIR/sidekiq-#{idx}"
        run "sv start #{sidekiq_dir}"
      end
    end

    desc "Restart sidekiq"
    task :restart, :roles => lambda { fetch(:sidekiq_role) }, :on_no_matching_servers => :continue do
      stop
      start
    end

  end
end
tippingpints
  • 1
  • 2
0

My solution for Capistrano 3:

set :sidekiq_runit_service_name, "sidekiq"
set :sidekiq_default_hooks, ->{ true }
set :sidekiq_cmd, ->{ "#{fetch(:bundle_cmd, "bundle")} exec sidekiq" }
set :sidekiqctl_cmd, ->{ "#{fetch(:bundle_cmd, "bundle")} exec sidekiqctl" }
set :sidekiq_timeout, ->{ 10 }
set :sidekiq_role,    ->{ :app }
set :sidekiq_pid,     ->{ "#{current_path}/tmp/pids/sidekiq.pid" }

namespace :sidekiq do

  desc "Quiet sidekiq (stop accepting new work)"
  task :quiet do
    on roles(fetch(:sidekiq_role)) do
      execute "if [ -d #{current_path} ] && [ -f #{fetch(:sidekiq_pid)} ] && kill -0 `cat #{fetch(:sidekiq_pid)}`> /dev/null 2>&1; then cd #{current_path} && #{fetch(:sidekiqctl_cmd)} quiet #{fetch(:sidekiq_pid)} ; else echo 'Sidekiq is not running'; fi"
    end
  end

  desc "Stop sidekiq"
  task :stop do
    on roles(fetch(:sidekiq_role)) do
      execute "sv stop #{fetch(:sidekiq_runit_service_name)}; true"
      execute "if [ -d #{current_path} ] && [ -f #{fetch(:sidekiq_pid)} ] && kill -0 `cat #{fetch(:sidekiq_pid)}`> /dev/null 2>&1; then cd #{current_path} && #{fetch(:sidekiqctl_cmd)} stop #{fetch(:sidekiq_pid)} #{fetch :sidekiq_timeout} ; else echo 'Sidekiq is not running'; fi"
    end
  end

  desc "Start sidekiq"
  task :start do
    on roles(fetch(:sidekiq_role)) do
      execute "sv start #{fetch(:sidekiq_runit_service_name)}"
    end
  end

  desc "Restart sidekiq"
  task :restart do
    invoke 'sidekiq:stop'
    invoke 'sidekiq:start'
  end

end

if fetch(:sidekiq_default_hooks)
  before 'deploy:started',   "sidekiq:quiet"
  after  'deploy:updated',   "sidekiq:stop"
  after  'deploy:reverted',  "sidekiq:stop"
  after  'deploy:published', "sidekiq:start"
  after  'deploy:restart',   "sidekiq:restart"
end
eagleas
  • 86
  • 6