Project

General

Profile

« Previous | Next » 

Revision a989a617

Added by Ohad Levy about 12 years ago

  • ID a989a6171e3130f50933e7d6694ab416c8f69cda

fixes #1509 - Foreman should use the proxy to import puppet classes

View differences:

app/models/environment.rb
has_many :hosts
validates_presence_of :name
validates_uniqueness_of :name
validates_format_of :name, :with => /^[\w\d]+$/, :message => "is alphanumeric and cannot contain spaces"
validates_format_of :name, :with => /^[\w\d]+$/, :message => "is alphanumeric and cannot contain spaces"
has_many :config_templates, :through => :template_combinations, :dependent => :destroy
has_many :template_combinations
......
name
end
# returns an hash of all puppet environments and their relative paths
def self.puppetEnvs
env = Hash.new
unless Rails.env == "test"
# reread puppet configuration
Puppet.clear
Puppet[:config] = SETTINGS[:puppetconfdir]
end
Puppet.parse_config # Check that puppet.conf has not been edited since the rack application was started
conf = Puppet.settings.instance_variable_get(:@values)
# query for the environments variable
unless conf[:main][:environments].nil?
conf[:main][:environments].split(",").each {|e| env[e.to_sym] = conf[e.to_sym][:modulepath] unless conf[e.to_sym][:modulepath].nil?}
else
# 0.25 doesn't require the environments variable anymore, scanning for modulepath
conf.keys.each {|p| env[p] = conf[p][:modulepath] unless conf[p][:modulepath].nil?}
# puppetmaster section "might" also returns the modulepath
env.delete :main
env.delete :puppetmasterd if env.size > 1
class << self
# returns an hash of all puppet environments and their relative paths
def puppetEnvs proxy = nil
#TODO: think of a better way to model multiple puppet proxies
url = (proxy || find_a_usable_proxy).try(:url)
raise "Can't find a valid Foreman Proxy with a Puppet feature" if url.blank?
proxy = ProxyAPI::Puppet.new :url => url
HashWithIndifferentAccess[proxy.environments.map { |e| [e, proxy.classes(e)] }]
end
if env.values.compact.size == 0
# fall back to defaults - we probably don't use environments
env[:production] = conf[:main][:modulepath] || conf[:puppetmasterd][:modulepath] || Setting[:modulepath]
end
return env
end
# Imports all Environments and classes from Puppet modules
def self.importClasses
# Build two hashes representing the on-disk and in-database, env to classes associations
disk_tree, db_tree = Hash.new([]), Hash.new([])
# Create a representation of the puppet configuration where the environments are hash keys and the classes are sorted lists
envs = self.puppetEnvs
for env, paths in envs
pclasses = []
for path in paths.split ":"
if Rails.env != "test"
# If we are deleting data then assure ourselves that we are using sensible values
raise "Unable to find directory #{path} in environment #{env}" unless File.directory?(path)
end
pclasses += Puppetclass.scanForClasses(path)
# Imports all Environments and classes from Puppet modules
def importClasses
# Build two hashes representing the on-disk and in-database, env to classes associations
# Create a representation of the puppet configuration where the environments are hash keys and the classes are sorted lists
disk_tree = puppetEnvs
disk_tree.default = []
# Create a representation of the foreman configuration where the environments are hash keys and the classes are sorted lists
db_tree = HashWithIndifferentAccess[Environment.all.map { |e| [e.name, e.puppetclasses.select(:name).map(&:name)] }]
db_tree.default = []
changes = { "new" => { }, "obsolete" => { } }
# Generate the difference between the on-disk and database configuration
for env in db_tree.keys
# Show the environment if there are classes in the db that do not exist on disk
# OR if there is no mention of the class on-disk
surplus_db_classes = db_tree[env] - disk_tree[env]
surplus_db_classes << "_destroy_" unless disk_tree.has_key?(env) # We need to distinguish between an empty and an obsolete env
changes["obsolete"][env] = surplus_db_classes if surplus_db_classes.size > 0
end
disk_tree[env.to_s] = pclasses.sort.uniq
end
# Create a representation of the foreman configuration where the environments are hash keys and the classes are sorted lists
for env in Environment.all
db_tree[env.name] = env.puppetclasses.map(&:name).sort.uniq
end
changes = {"new" => {}, "obsolete" => {}}
# Generate the difference between the on-disk and database configuration
for env in db_tree.keys
# Show the environment if there are classes in the db that do not exist on disk
# OR if there is no mention of the class on-disk
surplus_db_classes = db_tree[env] - disk_tree[env]
surplus_db_classes << "_destroy_" unless envs.has_key?(env.to_sym) # We need to distinguish between an empty and an obsolete env
changes["obsolete"][env] = surplus_db_classes if surplus_db_classes.size > 0
end
for env in disk_tree.keys
extra_disk_classes = disk_tree[env] - db_tree[env]
# Show the environment if there are new classes compared to the db
# OR if the environment has no puppetclasses but does not exist in the db
changes["new"][env] = extra_disk_classes if (extra_disk_classes.size > 0 or (disk_tree[env].size == 0 and Environment.find_by_name(env).nil?))
end
# Remove environments that are in config/ignored_environments.yml
ignored_file = File.join(Rails.root.to_s, "config", "ignored_environments.yml")
if File.exist? ignored_file
ignored = YAML.load_file ignored_file
for env in ignored[:new]
changes["new"].delete env
end
for env in ignored[:obsolete]
changes["obsolete"].delete env
for env in disk_tree.keys
extra_disk_classes = disk_tree[env] - db_tree[env]
# Show the environment if there are new classes compared to the db
# OR if the environment has no puppetclasses but does not exist in the db
changes["new"][env] = extra_disk_classes if (extra_disk_classes.size > 0 or (disk_tree[env].size == 0 and Environment.find_by_name(env).nil?))
end
end
changes
end
# Update the environments and puppetclasses based upon the user's selection
# It does a best attempt and can fail to perform all operations due to the
# user requesting impossible selections. Repeat the operation if errors are
# shown, after fixing the request.
# +changed+ : Hash with two keys: :new and :obsolete.
# changed[:/new|obsolete/] is and Array of Strings
# Returns : Array of Strings containing all record errors
def self.obsolete_and_new changed
changed ||= {}
@import_errors = []
# Now we add environments and associations
for env_str in changed[:new].keys
env = Environment.find_or_create_by_name env_str
if (env.valid? and ! env.new_record?)
begin
pclasses = eval(changed[:new][env_str])
rescue => e
@import_errors << "Failed to eval #{changed[:new][env_str]} as an array:" + e.message
next
# Remove environments that are in config/ignored_environments.yml
ignored_file = File.join(Rails.root.to_s, "config", "ignored_environments.yml")
if File.exist? ignored_file
ignored = YAML.load_file ignored_file
for env in ignored[:new]
changes["new"].delete env
end
for pclass in pclasses
pc = Puppetclass.find_or_create_by_name pclass
unless pc.errors.empty?
@import_errors += pc.errors.map(&:to_s)
else
env.puppetclasses << pc
end
for env in ignored[:obsolete]
changes["obsolete"].delete env
end
env.save!
else
@import_errors << "Unable to find or create environment #{env_str} in the foreman database"
end
end if changed[:new]
# Remove the obsoleted stuff
for env_str in changed[:obsolete].keys
env = Environment.find_by_name env_str
if env
begin
pclasses = eval(changed[:obsolete][env_str])
rescue => e
@import_errors << "Failed to eval #{changed[:obsolete][env_str]} as an array:" + e.message
next
end
pclass = ""
for pclass in pclasses
unless pclass == "_destroy_"
pc = Puppetclass.find_by_name pclass
if pc.nil?
@import_errors << "Unable to find puppet class #{pclass} in the foreman database"
changes
end
# Update the environments and puppetclasses based upon the user's selection
# It does a best attempt and can fail to perform all operations due to the
# user requesting impossible selections. Repeat the operation if errors are
# shown, after fixing the request.
# +changed+ : Hash with two keys: :new and :obsolete.
# changed[:/new|obsolete/] is and Array of Strings
# Returns : Array of Strings containing all record errors
def obsolete_and_new changed
changed ||= { }
@import_errors = []
# Now we add environments and associations
for env_str in changed[:new].keys
env = Environment.find_or_create_by_name env_str
if (env.valid? and !env.new_record?)
begin
pclasses = eval(changed[:new][env_str])
rescue => e
@import_errors << "Failed to eval #{changed[:new][env_str]} as an array:" + e.message
next
end
for pclass in pclasses
pc = Puppetclass.find_or_create_by_name pclass
unless pc.errors.empty?
@import_errors += pc.errors.map(&:to_s)
else
env.puppetclasses.delete pc
unless pc.environments.any? or pc.hosts.any?
pc.destroy
@import_errors += pc.errors.full_messages unless pc.errors.empty?
end
env.puppetclasses << pc
end
end
env.save!
else
@import_errors << "Unable to find or create environment #{env_str} in the foreman database"
end
if pclasses.include? "_destroy_"
env.destroy
@import_errors += env.errors.full_messages unless env.errors.empty?
end if changed[:new]
# Remove the obsoleted stuff
for env_str in changed[:obsolete].keys
env = Environment.find_by_name env_str
if env
begin
pclasses = eval(changed[:obsolete][env_str])
rescue => e
@import_errors << "Failed to eval #{changed[:obsolete][env_str]} as an array:" + e.message
next
end
pclass = ""
for pclass in pclasses
unless pclass == "_destroy_"
pc = Puppetclass.find_by_name pclass
if pc.nil?
@import_errors << "Unable to find puppet class #{pclass} in the foreman database"
else
env.puppetclasses.delete pc
unless pc.environments.any? or pc.hosts.any?
pc.destroy
@import_errors += pc.errors.full_messages unless pc.errors.empty?
end
end
end
end
if pclasses.include? "_destroy_"
env.destroy
@import_errors += env.errors.full_messages unless env.errors.empty?
else
env.save!
end
else
env.save!
@import_errors << "Unable to find environment #{env_str} in the foreman database"
end
else
@import_errors << "Unable to find environment #{env_str} in the foreman database"
end
end if changed[:obsolete]
end if changed[:obsolete]
@import_errors
end
@import_errors
end
def as_json(options={})
options ||= {}
super({:only => [:name, :id]}.merge(options))
end
private
private
def self.names_to_instances pcs, env
pcs.map do |pc|
if (theClass = Puppetclass.find_by_name(pc)).nil?
@import_errors << "Unable to add puppetclass '#{pc}' to #{env.name}. This is OK if you have disabled its creation'"
def find_a_usable_proxy
if (f = Feature.where(:name => "Puppet"))
if !f.empty? and (proxies=f.first.smart_proxies)
return proxies.first unless proxies.empty?
end
end
theClass
end.compact
nil
end
end
def as_json(options={ })
options ||= { }
super({ :only => [:name, :id] }.merge(options))
end
end
app/models/puppetclass.rb
name
end
# Scans a directory path for puppet classes
# +paths+ : String containing a colon separated module path
# returns
# Array of Strings containing puppet class names
def self.scanForClasses(paths)
klasses=Array.new
for path in paths.split(":")
Dir.glob("#{path}/*/manifests/**/*.pp").each do |manifest|
File.read(manifest).each_line do |line|
klass=line.match(/^class\s+([\w:-]*)/)
klasses << klass[1] if klass
end
end
end
return klasses.uniq
end
# returns a hash containing modules and associated classes
def self.classes2hash classes
hash = {}
......
klass <=> other.klass
end
# Retrieve the manifest dir from the puppet configuration
# Returns: String
def self.manifestdir
ps = Puppet.settings.instance_variable_get(:@values)
ps[:main][:manifestdir] || ps[:puppetmasterd][:manifestdir] || ps[:puppetd][:manifestdir] || Puppet.settings[:manifestdir] || "/etc/puppet/manifests"
end
# Populates the rdoc tree with information about all the classes in your modules.
# Firstly, we prepare the modules tree
# Secondly we run puppetdoc over the modulespath and manifestdir for all environments
app/models/smart_proxy.rb
class SmartProxy < ActiveRecord::Base
attr_accessible :name, :url
#TODO check if there is a way to look into the tftp_id too
# maybe with a perdefine sql
# maybe with a predefined sql
has_and_belongs_to_many :features
has_many :subnets, :foreign_key => "dhcp_id"
has_many :domains, :foreign_key => "dns_id"
lib/foreman/controller/environments.rb
module Foreman::Controller::Environments
def import_environments
@changed = Environment.importClasses
begin
@changed = Environment.importClasses
rescue => e
if e.message =~ /puppet feature/i
error "We did not find a foreman proxy that can provide the information, ensure that you have at least one Proxy with the puppet feature turned on."
redirect_to "/" + controller_path and return
else
raise e
end
end
if @changed["new"].size > 0 or @changed["obsolete"].size > 0
render "common/_puppetclasses_or_envs_changed"
else
notice "No changes to your environments detected"
redirect_to "/" + controller_path
end
rescue Exception => e
error e
redirect_to "/" + controller_path
end
def obsolete_and_new
if (errors = ::Environment.obsolete_and_new(params[:changed])).empty?
notice "Succcessfully updated environments and puppetclasses from the on-disk puppet installation"
notice "Successfully updated environments and puppetclasses from the on-disk puppet installation"
else
error "Failed to update the environments and puppetclasses from the on-disk puppet installation<br/>" + errors.join("<br>")
end
lib/proxy_api.rb
# TODO: add error message handling
def parse response
if response and response.code >= 200 and response.code < 300
return response.body.size > 2 ? JSON.parse(response.body) : true
return response.body.present? ? JSON.parse(response.body) : true
else
false
end
......
end
end
class Puppet < Resource
def initialize args
@url = args[:url] + "/puppet"
super args
end
def environments
parse(get "environments")
end
def environment env
parse(get "environments/#{env}")
end
def classes env
return if env.blank?
parse(get "environments/#{env}/classes").map { |k| k.keys.first }
rescue RestClient::ResourceNotFound
[]
end
end
class Puppetca < Resource
def initialize args
@url = args[:url] + "/puppet/ca"
......
end
def autosign
result = parse(get "autosign")
result == true ? [] : result
parse(get "autosign")
end
def set_autosign certname
lib/tasks/puppet.rake
exit
end
unless changes["new"].empty? and changes["obsolete"].empty?
unless changes["new"].empty? and changes["obsolete"].empty?
unless args.batch
puts "Scheduled changes to your environment"
puts "Create/update environments"
test/fixtures/features.yml
name: Puppet CA
puppet:
name: puppet
name: Puppet
test/fixtures/smart_proxies.yml
puppetmaster:
name: Puppetmaster Proxy
url: http://else.where:4567
features: puppet, 'puppet ca'
test/functional/environments_controller_test.rb
end
# This is the on-disk status
# and should result in a disk_tree of {"env1" => ["a", "b", "c"],"env2" => ["a", "b", "c"]}
Environment.expects(:puppetEnvs).returns(:env1 => "/etc/puppet/env/muc",
:env2 => "/etc/puppet/env/muc"
).at_least_once
Puppetclass.expects(:scanForClasses).returns(["a", "b", "c"]).at_least_once
envs = HashWithIndifferentAccess.new(:env1 => %w{a b c}, :env2 => %w{a b c})
Environment.expects(:puppetEnvs).returns(envs).at_least_once
end
test "should handle disk environment containing additional classes" do
setup_import_classes
Environment.find_by_name("env1").puppetclasses.delete(Puppetclass.find_by_name("a"))
#db_tree of {"env1" => ["b", "c"], "env2" => ["a", "b", "c"]}
#disk_tree of {"env1" => ["a", "b", "c"],"env2" => ["a", "b", "c"]}
# db_tree of {"env1" => ["b", "c"], "env2" => ["a", "b", "c"]}
# disk_tree of {"env1" => ["a", "b", "c"],"env2" => ["a", "b", "c"]}
get :import_environments, {}, set_session_user
assert_template "puppetclasses_or_envs_changed"
assert_select 'input#changed_new_env1[value*="a"]'
......
}
}, set_session_user
assert_redirected_to environments_url
assert flash[:notice] = "Succcessfully updated environments and puppetclasses from the on-disk puppet installation"
assert flash[:notice] = "Successfully updated environments and puppetclasses from the on-disk puppet installation"
assert Environment.find_by_name("env1").puppetclasses.map(&:name).sort == ["a", "b", "c"]
end
test "should handle disk environment containing less classes" do
......
}
}, set_session_user
assert_redirected_to environments_url
assert flash[:notice] = "Succcessfully updated environments and puppetclasses from the on-disk puppet installation"
assert flash[:notice] = "Successfully updated environments and puppetclasses from the on-disk puppet installation"
assert Environment.find_by_name("env3").puppetclasses.map(&:name).sort == []
end
......
setup_import_classes
as_admin do
Environment.create :name => "env3"
Environment.where(:name => "env1").first.delete
Environment.delete_all(:name => "env1")
end
#db_tree of { , "env2" => ["a", "b", "c"], "env3" => []}
#disk_tree of {"env1" => ["a", "b", "c"], "env2" => ["a", "b", "c"]}
test/test_helper.rb
Net::DNS::PTRRecord.any_instance.stubs(:conflicting?).returns(false)
Net::DHCP::Record.any_instance.stubs(:create).returns(true)
Net::DHCP::SparcRecord.any_instance.stubs(:create).returns(true)
ProxyAPI::Puppet.any_instance.stubs(:environments).returns(["production"])
end
def disable_orchestration
test/unit/environment_test.rb
test "to_label should print name" do
env = Environment.new :name => "foo"
assert env.to_label == env.name
assert_equal env.to_label, env.name
end
test "to_s should print name" do
env = Environment.new :name => "foo"
assert env.to_s == env.name
end
test "with Puppet previous to 0.25, self.puppetEnvs should import environments" do
Puppet.settings.instance_variable_get(:@values)[:main][:environments] = "test,development,production"
Puppet.settings.instance_variable_get(:@values)[:test][:modulepath] = "/test/some/path"
Puppet.settings.instance_variable_get(:@values)[:development][:modulepath] = "/development/some/path"
Puppet.settings.instance_variable_get(:@values)[:production][:modulepath] = "/production/some/path"
Puppet.settings.expects(:parse) # puppetEnvs now reparses the file so we need to stub that function
environments = Environment.puppetEnvs
assert_not_nil environments[:test]
assert environments[:test] == "/test/some/path"
assert_not_nil environments[:development]
assert environments[:development] == "/development/some/path"
assert_not_nil environments[:production]
assert environments[:production] == "/production/some/path"
end
test "with Puppet later than 0.25, self.puppetEnvs should import environments" do
Puppet.settings.instance_variable_get(:@values)[:main] = {}
Puppet.settings.instance_variable_get(:@values)[:puppetmasterd] = {}
Puppet.settings.instance_variable_get(:@values)[:test] = {:modulepath => "/test/some/path"}
Puppet.settings.instance_variable_get(:@values)[:development] = {:modulepath => "/development/some/path"}
Puppet.settings.instance_variable_get(:@values)[:production] = {:modulepath => "/production/some/path"}
Puppet.settings.expects(:parse) # puppetEnvs now reparses the file so we need to stub that function
environments = Environment.puppetEnvs
assert_nil environments[:main]
assert_nil environments[:puppetmasterd]
assert_not_nil environments[:test]
assert environments[:test] == "/test/some/path"
assert_not_nil environments[:development]
assert environments[:development] == "/development/some/path"
assert_not_nil environments[:production]
assert environments[:production] == "/production/some/path"
end
test "if no env is defined by Puppet, self.puppetEnvs should define production" do
Puppet.settings.instance_variable_get(:@values).clear
Puppet.settings.expects(:parse) # puppetEnvs now reparses the file so we need to stub that function
environments = Environment.puppetEnvs
assert_not_nil environments[:production]
assert_equal env.to_s, env.name
end
end
test/unit/puppetclass_test.rb
assert !other_puppet_class.save
end
test "scanForClasses should retrieve puppetclasses from .pp files" do
path = "/some/path"
puppet_classes = ["class some_puppet_class {","class other_puppet_class{","class yet_another_puppet_class{"]
mock(Dir).glob("#{path}/*/manifests/**/*.pp") { puppet_classes }
puppet_classes.each do |puppet_class|
mock(File).read(anything) { StringIO.new(puppet_class) }
end
klasses = Puppetclass.scanForClasses path
assert klasses[0] == "some_puppet_class"
assert klasses[1] == "other_puppet_class"
assert klasses[2] == "yet_another_puppet_class"
end
def setup_user operation
@one = users(:one)
as_admin do

Also available in: Unified diff