Project

General

Profile

« Previous | Next » 

Revision 35c14172

Added by Ivan Necas over 7 years ago

Refs #15779 - make background processing unavailable for now (#4217)

The original PR got vetted in the packaging phase and including
foreman-task as dependency of Foreman was refused. We need to rethink
our approach for getting the foreman-tasks functionality available
inside the core and take less obtrusive approach by making the
transition in several steps, until the foreman core will get on
future-parity with foreman-tasks which would effectively mean tasks in
core.

To clean the tasks that were created while using the original changes,
you can run:

rake foreman_tasks:cleanup\
    TASK_SEARCH="label = Actions::Foreman::Report::Import
      OR label = Actions::Foreman::PuppetClass::Import"\
    VERBOSE=true

This reverts commits

eb371ca33dcdd7a4f71d3cda0cfcad701cd3ae7e,
     69c34cd691aea4c7f21613f3859ae92a5f403529,
     3a1cd1dbeb74c60a482d7c47fc290c185d7c707a,
     0cf8ea6780c24954ccbe2cc33ca02e1f6010ffed,
     1538b8d2addb12f04da36a2c34224cd821e77acc,
     33709ef2813d0128f64905c6597616f44abad28e.

View differences:

app/controllers/api/base_controller.rb
end
end
def process_success(response = nil, render_status = nil)
render_status ||= request.post? ? :created : :ok
def process_success(response = nil)
render_status = request.post? ? :created : :ok
response ||= get_resource
respond_with response, :responder => ApiResponder, :status => render_status
end
app/controllers/api/v2/config_reports_controller.rb
param_group :config_report, :as => :create
def create
import_params = { :report => params[:config_report] }
task = ForemanTasks.async_task Actions::Foreman::Report::Import, import_params, ConfigReport, detected_proxy.try(:id)
@message = _('Report import has been enqueued, please see the task %s for more details') % url_for(task)
process_success @message, :accepted
@config_report = ConfigReport.import(params[:config_report], detected_proxy.try(:id))
process_response @config_report.errors.empty?
rescue ::Foreman::Exception => e
render_message(e.to_s, :status => :unprocessable_entity)
end
app/controllers/api/v2/reports_controller.rb
param_group :report, :as => :create
def create
import_params = { :report => params[:report] }
task = ForemanTasks.async_task Actions::Foreman::Report::Import, import_params, resource_class, detected_proxy.try(:id)
@message = _('Report import has been enqueued, please see the task %s for more details') % url_for(task)
process_success @message, :accepted
@report = resource_class.import(params[:report], detected_proxy.try(:id))
process_response @report.errors.empty?
rescue ::Foreman::Exception => e
render_message(e.to_s, :status => :unprocessable_entity)
end
app/controllers/concerns/api/import_puppetclasses_common_controller.rb
param :smart_proxy_id, String, :required => false
param :environment_id, String, :required => false
param :dryrun, :bool, :required => false
param :background, :bool, :required => false
param :except, String, :required => false, :desc => N_("Optional comma-delimited string containing either 'new', 'updated', or 'obsolete' that is used to limit the imported Puppet classes")
def import_puppetclasses
......
end
# RUN PuppetClassImporter
background = params.key?(:background) && !['false', false].include?(params[:background])
begin
task = ForemanTasks.trigger_task(background, ::Actions::Foreman::PuppetClass::Import, :changed => @changed)
if background
process_success task
else
render("api/v#{api_version}/import_puppetclasses/#{rabl_template}", :layout => "api/layouts/import_puppetclasses_layout")
end
rescue ForemanTasks::TaskError => e
render :json => { :message => _("Failed to update the environments and Puppet classes from the on-disk puppet installation: %s") % e.to_s }, :status => :internal_server_error
if (errors = ::PuppetClassImporter.new.obsolete_and_new(@changed)).empty?
render("api/v#{api_version}/import_puppetclasses/#{rabl_template}", :layout => "api/layouts/import_puppetclasses_layout")
else
render :json => {:message => _("Failed to update the environments and Puppet classes from the on-disk puppet installation: %s") % errors.join(", ")}, :status => :internal_server_error
end
end
app/controllers/concerns/foreman/controller/environments.rb
end
def obsolete_and_new
import_params = { :changed => params[:changed] }
if params[:commit] == _("Update on background")
ForemanTasks.async_task(::Actions::Foreman::PuppetClass::Import, import_params)
notice _("Added import task to the queue, it will be run shortly")
if (errors = ::PuppetClassImporter.new.obsolete_and_new(params[:changed])).empty?
notice _("Successfully updated environments and Puppet classes from the on-disk Puppet installation")
else
ForemanTasks.sync_task(::Actions::Foreman::PuppetClass::Import, import_params)
begin
notice _('Successfully updated environments and Puppet classes from the on-disk Puppet installation')
rescue ForemanTasks::TaskError
error _('Failed to update environments and Puppet classes from the Puppet installation')
end
error _("Failed to update environments and Puppet classes from the on-disk Puppet installation: %s") % errors.to_sentence
end
rescue ::Foreman::Exception => e
error _("Failed to add task to queue: %s") % e.to_s
ensure
redirect_to :controller => controller_path
end
app/models/config_report.rb
"status"
end
def self.humanized_name
N_('Config report')
end
# a method that save the report values (e.g. values from METRIC)
# it is not supported to edit status values after it has been written once.
def status=(st)
app/models/report.rb
class Report < ActiveRecord::Base
LOG_LEVELS = %w[debug info notice warning err alert emerg crit]
DEFAULT_EXPIRATION = 1.week
include Foreman::STI
include Authorizable
......
# with_changes
scope :interesting, -> { where("status <> 0") }
def self.humanized_name
N_('Report')
end
# extracts serialized metrics and keep them as a hash_with_indifferent_access
def metrics
return {} if read_attribute(:metrics).nil?
......
# Expire reports based on time and status
# Defaults to expire reports older than a week regardless of the status
def self.expire(conditions = {})
timerange = conditions[:timerange] || DEFAULT_EXPIRATION
timerange = conditions[:timerange] || 1.week
status = conditions[:status]
cond = "reports.created_at < \'#{(Time.now.utc - timerange).to_formatted_s(:db)}\'"
cond += " and reports.status = #{status}" unless status.nil?
app/services/actions/foreman/puppet_class/import.rb
module Actions
module Foreman
module PuppetClass
class Import < Actions::EntryAction
def resource_locks
:import_puppetclasses
end
def run
# #obsolete_and_new can return nil if there's no change so we have to be careful with to_sentence
output[:errors] = ::PuppetClassImporter.new.obsolete_and_new(input[:changed]).try(:to_sentence)
end
def humanized_output
return nil if input[:changed].nil?
humanized_output = []
humanized_output << _('Add') + ' ' + format_env_and_classes_input(input[:changed][:new]) if input[:changed][:new].present?
humanized_output << _('Remove') + ' ' + format_env_and_classes_input(input[:changed][:obsolete]) if input[:changed][:obsolete].present?
humanized_output << _('Update') + ' ' + format_env_and_classes_input(input[:changed][:updated]) if input[:changed][:updated].present?
humanized_output.join("\n")
end
def rescue_strategy
::Dynflow::Action::Rescue::Skip
end
def humanized_name
_("Import Environments and Puppet classes")
end
# default value for cleaning up the tasks, it can be overriden by settings
def self.cleanup_after
'30d'
end
private
def format_env_and_classes_input(selection)
selection.map do |environment, classes|
result = _('environment') + " #{environment}"
classes = JSON.parse(classes)
no_classes_info = classes.include?('_destroy_')
result += " (#{classes.size} " + _('classes') + ")" unless no_classes_info
result
end.join(', ')
end
end
end
end
end
app/services/actions/foreman/report/import.rb
module Actions
module Foreman
module Report
class Import < Actions::EntryAction
def resource_locks
:import_reports
end
def plan(params, report_class, detected_proxy_id)
plan_self :params => params, :report_class => report_class.to_s, :detected_proxy_id => detected_proxy_id
end
def run
report_class = input[:report_class].constantize
report = report_class.import(input[:params][:report], SmartProxy.find_by_id(input[:detected_proxy_id]))
if report.errors.any?
raise _('Failed importing of report: %s') % report.errors.full_messages
else
output[:report_id] = report.id
end
end
def rescue_strategy
::Dynflow::Action::Rescue::Skip
end
def humanized_name
N_("Import")
end
def humanized_input
input[:report_class].constantize.humanized_name
end
def self.cleanup_after
"#{::Report::DEFAULT_EXPIRATION / 1.day}d"
end
end
end
end
end
app/validators/bookmark_controller_validator.rb
class BookmarkControllerValidator < ActiveModel::EachValidator
@@active_record_tables = ActiveRecord::Base.connection.tables.map(&:to_s)
def validate_each(record, attribute, value)
controllers = ["dashboard"] + (active_record_tables + Permission.resources.map {|x| x.tableize }).uniq
controllers = ["dashboard"] + (@@active_record_tables + Permission.resources.map {|x| x.tableize }).uniq
record.errors[attribute] << _("%{value} is not a valid controller") % {:value => value } unless controllers.include?(value)
end
private
def active_record_tables
ActiveRecord::Base.connection.tables.map(&:to_s)
end
end
app/views/api/v2/config_reports/create.json.rabl
object @config_report
extends "api/v2/config_reports/show"
node(:message) { @message }
app/views/api/v2/reports/create.json.rabl
object @report
extends "api/v2/reports/show"
node(:message) { @message }
app/views/common/_puppetclasses_or_envs_changed.html.erb
<div>
<%= link_to _("Cancel"), send("#{controller_name}_path"), :class => "btn btn-default" %>
<%= submit_tag _("Update"), :class => "btn btn-primary" %>
<%= submit_tag _("Update on background"), :class => "btn btn-primary" %>
</div>
<% end %>
bundler.d/foreman_tasks.rb
group :foreman_tasks do
gem 'foreman-tasks', '>= 0.8.5'
gem 'concurrent-ruby-edge', '0.2.3'
end
config/application.rb
end
end
end
begin
Bundler.require(:foreman_tasks)
rescue LoadError => e
warn "Could not load foreman tasks, async processing will not be available: #{e.message}"
end
end
end
test/controllers/api/v1/smart_proxies_controller_test.rb
require 'test_helper'
class Api::V1::SmartProxiesControllerTest < ActionController::TestCase
include ForemanTasks::TestHelpers::WithInThreadExecutor
valid_attrs = { :name => 'master02', :url => 'http://server:8443' }
setup do
test/controllers/api/v2/config_reports_controller_test.rb
require 'controllers/shared/report_host_permissions_test'
class Api::V2::ConfigReportsControllerTest < ActionController::TestCase
include ForemanTasks::TestHelpers::WithInThreadExecutor
include ::ReportHostPermissionsTest
describe "Non Admin User" do
......
def test_create_invalid
User.current=nil
post :create, {:config_report => ["not a hash", "throw an error"] }, set_session_user
assert_response :success
assert_response :unprocessable_entity
end
def test_create_duplicate
......
post :create, {:config_report => create_a_puppet_transaction_report }, set_session_user
assert_response :success
post :create, {:config_report => create_a_puppet_transaction_report }, set_session_user
assert_response :success
assert_response :unprocessable_entity
end
test 'when ":restrict_registered_smart_proxies" is false, HTTP requests should be able to create a report' do
......
Resolv.any_instance.stubs(:getnames).returns(['else.where'])
post :create, {:config_report => create_a_puppet_transaction_report }
assert_nil @controller.detected_proxy
assert_response :accepted
assert_response :created
end
test 'hosts with a registered smart proxy on should create a report successfully' do
......
Resolv.any_instance.stubs(:getnames).returns([host])
post :create, {:config_report => create_a_puppet_transaction_report }
assert_equal proxy, @controller.detected_proxy
assert_response :accepted
assert_response :created
end
test 'hosts without a registered smart proxy on should not be able to create a report' do
......
@request.env['SSL_CLIENT_S_DN'] = 'CN=else.where'
@request.env['SSL_CLIENT_VERIFY'] = 'SUCCESS'
post :create, {:config_report => create_a_puppet_transaction_report }
assert_response :accepted
assert_response :created
end
test 'hosts without a registered smart proxy but with an SSL cert should not be able to create a report' do
......
Resolv.any_instance.stubs(:getnames).returns(['else.where'])
post :create, {:config_report => create_a_puppet_transaction_report }
assert_response :accepted
assert_response :created
end
end
test/controllers/api/v2/reports_controller_test.rb
require 'controllers/shared/report_host_permissions_test'
class Api::V2::ReportsControllerTest < ActionController::TestCase
include ForemanTasks::TestHelpers::WithInThreadExecutor
include ::ReportHostPermissionsTest
setup do
......
def test_create_valid
User.current=nil
post :create, {:report => create_a_puppet_transaction_report }, set_session_user
assert_response :accepted
assert_response :success
end
def test_create_invalid
User.current=nil
post :create, {:report => ["not a hash", "throw an error"] }, set_session_user
assert_response :accepted
assert_response :unprocessable_entity
end
def test_create_duplicate
User.current=nil
post :create, {:report => create_a_puppet_transaction_report }, set_session_user
assert_response :accepted
assert_response :success
Foreman::Deprecation.expects(:api_deprecation_warning)
post :create, {:report => create_a_puppet_transaction_report }, set_session_user
assert_response :accepted
assert_response :unprocessable_entity
end
test 'when ":restrict_registered_smart_proxies" is false, HTTP requests should be able to create a report' do
......
Resolv.any_instance.stubs(:getnames).returns(['else.where'])
post :create, {:report => create_a_puppet_transaction_report }
assert_nil @controller.detected_proxy
assert_response :accepted
assert_response :created
end
test 'hosts with a registered smart proxy on should create a report successfully' do
......
Resolv.any_instance.stubs(:getnames).returns([host])
post :create, {:report => create_a_puppet_transaction_report }
assert_equal proxy, @controller.detected_proxy
assert_response :accepted
assert_response :created
end
test 'hosts without a registered smart proxy on should not be able to create a report' do
......
@request.env['SSL_CLIENT_S_DN'] = 'CN=else.where'
@request.env['SSL_CLIENT_VERIFY'] = 'SUCCESS'
post :create, {:report => create_a_puppet_transaction_report }
assert_response :accepted
assert_response :created
end
test 'hosts without a registered smart proxy but with an SSL cert should not be able to create a report' do
......
Resolv.any_instance.stubs(:getnames).returns(['else.where'])
post :create, {:report => create_a_puppet_transaction_report }
assert_response :accepted
assert_response :created
end
end
test/controllers/api/v2/smart_proxies_controller_test.rb
require 'test_helper'
class Api::V2::SmartProxiesControllerTest < ActionController::TestCase
include ForemanTasks::TestHelpers::WithInThreadExecutor
valid_attrs = { :name => 'master02', :url => 'http://server:8443' }
setup do
test/controllers/api/v2/template_combinations_controller_test.rb
require 'test_helper'
class Api::V2::TemplateCombinationsControllerTest < ActionController::TestCase
include ForemanTasks::TestHelpers::WithInThreadExecutor
context 'with provisioning_template_id' do
setup do
Foreman::Deprecation.expects(:api_deprecation_warning).never
test/controllers/config_reports_controller_test.rb
require 'controllers/shared/report_host_permissions_test'
class ConfigReportsControllerTest < ActionController::TestCase
include ForemanTasks::TestHelpers::WithInThreadExecutor
include ::ReportHostPermissionsTest
def test_index
test/controllers/environments_controller_test.rb
require 'test_helper'
class EnvironmentsControllerTest < ActionController::TestCase
include ForemanTasks::TestHelpers::WithInThreadExecutor
setup do
@model = Environment.first
end
test/controllers/puppetclasses_controller_test.rb
require 'test_helper'
class PuppetclassesControllerTest < ActionController::TestCase
include ForemanTasks::TestHelpers::WithInThreadExecutor
include LookupKeysHelper
def host_attributes(host)
test/integration_test_helper.rb
class IntegrationTestWithJavascript < ActionDispatch::IntegrationTest
def login_admin
DatabaseCleaner.strategy = :truncation, { :except => ['dynflow_schema_info'] }
DatabaseCleaner.strategy = :truncation
DatabaseCleaner.start
Capybara.current_driver = Capybara.javascript_driver
super
test/test_helper.rb
require 'controllers/shared/basic_rest_response_test'
require 'facet_test_helper'
require 'active_support_test_case_helper'
require 'foreman_tasks/test_helpers'
ForemanTasks.dynflow.require!
ForemanTasks.dynflow.config.disable_active_record_actions = true
Shoulda::Matchers.configure do |config|
config.integrate do |with|
test/unit/actions/report/import_test.rb
require 'test_helper'
require 'dynflow/testing'
module Actions
module Foreman
module Report
class ImportTest < ActiveSupport::TestCase
class DummyReportClass
def self.import(*args)
end
end
include Dynflow::Testing
let(:action) do
create_action(Actions::Foreman::Report::Import)
end
let(:planned) do
plan_action action, {}, DummyReportClass, FactoryGirl.create(:smart_proxy).id
end
describe 'importing' do
it 'calls import on report class' do
DummyReportClass.expects(:import).returns(OpenStruct.new(:errors => []))
run_action planned
end
it 'raises exception if some error exists' do
DummyReportClass.expects(:import).returns(OpenStruct.new(:errors => OpenStruct.new(:any? => true, :full_messages => 'custom string')))
exception = assert_raises(RuntimeError) { run_action planned }
assert_includes exception.message, 'custom string'
end
end
end
describe 'cleanup' do
it 'derive the number of dates based on Report::DEFAULT_EXPIRATION' do
assert_equal '7d', Actions::Foreman::Report::Import.cleanup_after
end
end
end
end
end
test/unit/tasks/seeds_test.rb
self.use_transactional_fixtures = false
setup do
DatabaseCleaner.clean_with :truncation, :except => ['dynflow_schema_info']
DatabaseCleaner.clean_with :truncation
Setting.stubs(:[]).with(:administrator).returns("root@localhost")
Setting.stubs(:[]).with(:send_welcome_email).returns(false)
Foreman.stubs(:in_rake?).returns(true)

Also available in: Unified diff