foreman/app/models/orchestration.rb @ ff61accf
| 2a0cffd3 | Ohad Levy | require_dependency "proxy_api"
|
|
| 6454273f | Paul Kelly | require 'orchestration/queue'
|
|
| 90b83222 | Ohad Levy | ||
module Orchestration
|
|||
def self.included(base)
|
|||
base.send :include, InstanceMethods
|
|||
base.class_eval do
|
|||
| c7540a9c | Amos Benari | attr_reader :queue, :post_queue, :old, :record_conflicts
|
|
| 90b83222 | Ohad Levy | # stores actions to be performed on our proxies based on priority
|
|
before_validation :set_queue
|
|||
before_validation :setup_clone
|
|||
# extend our Host model to know how to handle subsystems
|
|||
| ac36e7ce | Ohad Levy | include Orchestration::DNS
|
|
| 30ae12bf | Ohad Levy | include Orchestration::DHCP
|
|
| 90b83222 | Ohad Levy | include Orchestration::TFTP
|
|
| 36f93e4d | Ohad Levy | include Orchestration::Puppetca
|
|
| 96be8845 | Ohad Levy | include Orchestration::Libvirt
|
|
| 334d0359 | Amos Benari | include Orchestration::Compute
|
|
| dd42df0a | Ohad Levy | include Orchestration::SSHProvision
|
|
| 90b83222 | Ohad Levy | ||
# save handles both creation and update of hosts
|
|||
before_save :on_save
|
|||
| c7540a9c | Amos Benari | after_commit :post_commit
|
|
| 822eef5a | Paul Kelly | after_destroy :on_destroy
|
|
| 90b83222 | Ohad Levy | end
|
|
end
|
|||
module InstanceMethods
|
|||
protected
|
|||
def on_save
|
|||
| 0363ab42 | Amos Benari | process :queue
|
|
| 90b83222 | Ohad Levy | end
|
|
| c7540a9c | Amos Benari | def post_commit
|
|
| 0363ab42 | Amos Benari | process :post_queue
|
|
| c7540a9c | Amos Benari | end
|
|
| 822eef5a | Paul Kelly | def on_destroy
|
|
| 0363ab42 | Amos Benari | errors.empty? ? process(:queue) : false
|
|
| 822eef5a | Paul Kelly | end
|
|
| 90b83222 | Ohad Levy | def rollback
|
|
raise ActiveRecord::Rollback
|
|||
end
|
|||
# log and add to errors
|
|||
| 30ae12bf | Ohad Levy | def failure msg, backtrace=nil, dest = :base
|
|
| e85584bf | Paul Kelly | logger.warn(backtrace ? msg + backtrace.join("\n") : msg)
|
|
| 30ae12bf | Ohad Levy | errors.add dest, msg
|
|
| 90b83222 | Ohad Levy | false
|
|
end
|
|||
public
|
|||
# we override this method in order to include checking the
|
|||
# after validation callbacks status, as rails by default does
|
|||
# not care about their return status.
|
|||
| 017e1049 | Ohad Levy | def valid?(context = nil)
|
|
| 90b83222 | Ohad Levy | super
|
|
| 30ae12bf | Ohad Levy | orchestration_errors?
|
|
| 90b83222 | Ohad Levy | end
|
|
# we override the destroy method, in order to ensure our queue exists before other callbacks
|
|||
# and to process the queue only if we found no errors
|
|||
def destroy
|
|||
set_queue
|
|||
super
|
|||
end
|
|||
private
|
|||
| 173724cc | Ohad Levy | ||
| e85584bf | Paul Kelly | def proxy_error e
|
|
| 173724cc | Ohad Levy | e.respond_to?(:message) ? e.message : e
|
|
| e85584bf | Paul Kelly | end
|
|
| 90b83222 | Ohad Levy | # Handles the actual queue
|
|
# takes care for running the tasks in order
|
|||
# if any of them fail, it rollbacks all completed tasks
|
|||
# in order not to keep any left overs in our proxies.
|
|||
| 0363ab42 | Amos Benari | def process queue_name
|
|
| 90b83222 | Ohad Levy | return true if Rails.env == "test"
|
|
# queue is empty - nothing to do.
|
|||
| 0363ab42 | Amos Benari | q = send(queue_name)
|
|
| 90b83222 | Ohad Levy | return if q.empty?
|
|
# process all pending tasks
|
|||
q.pending.each do |task|
|
|||
# if we have failures, we don't want to process any more tasks
|
|||
| c7c5d840 | Paul Kelly | break unless q.failed.empty?
|
|
| 90b83222 | Ohad Levy | task.status = "running"
|
|
| dd42df0a | Ohad Levy | ||
| 0363ab42 | Amos Benari | update_cache
|
|
| 90b83222 | Ohad Levy | begin
|
|
task.status = execute({:action => task.action}) ? "completed" : "failed"
|
|||
| a6f4f5f7 | Ohad Levy | rescue Net::Conflict => e
|
|
task.status = "conflict"
|
|||
@record_conflicts << e
|
|||
| 30ae12bf | Ohad Levy | failure e.message, nil, :conflict
|
|
#TODO: This is not a real error, but at the moment the proxy / foreman lacks better handling
|
|||
# of the error instead of explode.
|
|||
rescue Net::LeaseConflict => e
|
|||
task.status = "failed"
|
|||
failure "DHCP has a lease at #{e}"
|
|||
| c7c5d840 | Paul Kelly | rescue RestClient::Exception => e
|
|
task.status = "failed"
|
|||
| 26e2d30d | Amos Benari | failure "#{task.name} task failed with the following error: #{proxy_error e}"
|
|
| 90b83222 | Ohad Levy | rescue => e
|
|
task.status = "failed"
|
|||
| f28a6895 | Ohad Levy | failure "#{task.name} task failed with the following error: #{e}"
|
|
| 90b83222 | Ohad Levy | end
|
|
end
|
|||
| 0363ab42 | Amos Benari | update_cache
|
|
| 90b83222 | Ohad Levy | # if we have no failures - we are done
|
|
| 30ae12bf | Ohad Levy | return true if q.failed.empty? and q.pending.empty? and q.conflict.empty? and orchestration_errors?
|
|
| 90b83222 | Ohad Levy | ||
| f28a6895 | Ohad Levy | logger.warn "Rolling back due to a problem: #{q.failed + q.conflict}"
|
|
| dd42df0a | Ohad Levy | q.pending.each{ |task| task.status = "canceled" }
|
|
| 90b83222 | Ohad Levy | # handle errors
|
|
# we try to undo all completed operations and trigger a DB rollback
|
|||
| 102a8035 | Paul Kelly | (q.completed + q.running).sort.reverse_each do |task|
|
|
| 90b83222 | Ohad Levy | begin
|
|
task.status = "rollbacked"
|
|||
| 0363ab42 | Amos Benari | update_cache
|
|
| 90b83222 | Ohad Levy | execute({:action => task.action, :rollback => true})
|
|
rescue => e
|
|||
# if the operation failed, we can just report upon it
|
|||
| a6f4f5f7 | Ohad Levy | failure "Failed to perform rollback on #{task.name} - #{e}"
|
|
| 90b83222 | Ohad Levy | end
|
|
end
|
|||
rollback
|
|||
end
|
|||
def execute opts = {}
|
|||
obj, met = opts[:action]
|
|||
rollback = opts[:rollback] || false
|
|||
# at the moment, rollback are expected to replace set with del in the method name
|
|||
if rollback
|
|||
met = met.to_s
|
|||
case met
|
|||
when /set/
|
|||
met.gsub!("set","del")
|
|||
when /del/
|
|||
met.gsub!("del","set")
|
|||
else
|
|||
raise "Dont know how to rollback #{met}"
|
|||
end
|
|||
met = met.to_sym
|
|||
end
|
|||
if obj.respond_to?(met)
|
|||
return obj.send(met)
|
|||
else
|
|||
| a6f4f5f7 | Ohad Levy | failure "invalid method #{met}"
|
|
| 90b83222 | Ohad Levy | raise "invalid method #{met}"
|
|
end
|
|||
end
|
|||
def set_queue
|
|||
| 6454273f | Paul Kelly | @queue = Orchestration::Queue.new
|
|
| c7540a9c | Amos Benari | @post_queue = Orchestration::Queue.new
|
|
| a6f4f5f7 | Ohad Levy | @record_conflicts = []
|
|
| 90b83222 | Ohad Levy | end
|
|
# we keep the before update host object in order to compare changes
|
|||
def setup_clone
|
|||
return if new_record?
|
|||
| ff61accf | Ohad Levy | @old = dup
|
|
| 90b83222 | Ohad Levy | for key in (changed_attributes.keys - ["updated_at"])
|
|
@old.send "#{key}=", changed_attributes[key]
|
|||
| 672f931d | Paul Kelly | # At this point the old cached bindings may still be present so we force an AR association reload
|
|
# This logic may not work or be required if we switch to Rails 3
|
|||
if (match = key.match(/^(.*)_id$/))
|
|||
name = match[1].to_sym
|
|||
next if name == :owner # This does not work for the owner association even from the console
|
|||
self.send(name, true) if (send(name) and send(name).id != @attributes[key])
|
|||
old.send(name, true) if (old.send(name) and old.send(name).id != old.attributes[key])
|
|||
end
|
|||
| 90b83222 | Ohad Levy | end
|
|
end
|
|||
| 30ae12bf | Ohad Levy | ||
def orchestration_errors?
|
|||
overwrite? ? errors.are_all_conflicts? : errors.empty?
|
|||
end
|
|||
| 0363ab42 | Amos Benari | def update_cache
|
|
| ff217463 | Amos Benari | Rails.cache.write(progress_report_id, (queue.all + post_queue.all).to_json, :expires_in => 5.minutes)
|
|
| 0363ab42 | Amos Benari | end
|
|
| 90b83222 | Ohad Levy | end
|
|
end
|