foreman/lib/tasks/convert.rake @ 9d43fc71
a53137f7 | Ohad Levy | #
|
|
# Convert/transfer data from production => development. This facilitates
|
|||
# a conversion one database adapter type to another (say postgres -> mysql )
|
|||
#
|
|||
# WARNING 1: this script deletes all development data and replaces it with
|
|||
# production data
|
|||
#
|
|||
# WARNING 2: This script assumes it is the only user updating either database.
|
|||
# Database integrity could be corrupted if other users where
|
|||
# writing to the databases.
|
|||
#
|
|||
8a5d3250 | Ohad Levy | # Usage: rake db:convert:prod2dev
|
|
#
|
|||
# It assumes the development database has a schema identical to the production
|
|||
# database, but will delete any data before importing the production data
|
|||
#
|
|||
# A couple of the outer loops evolved from
|
|||
# http://snippets.dzone.com/posts/show/3393
|
|||
#
|
|||
# For further instructions see
|
|||
# http://myutil.com/2008/8/31/rake-task-transfer-rails-database-mysql-to-postgres
|
|||
#
|
|||
# The master repository for this script is at github:
|
|||
# http://github.com/face/rails_db_convert_using_adapters/tree/master
|
|||
#
|
|||
# Author: Rama McIntosh
|
|||
# Matson Systems, Inc.
|
|||
# http://www.matsonsystems.com
|
|||
#
|
|||
# This rake task is released under this BSD license:
|
|||
#
|
|||
# Copyright (c) 2008, Matson Systems, Inc. All rights reserved.
|
|||
#
|
|||
# Redistribution and use in source and binary forms, with or without
|
|||
# modification, are permitted provided that the following conditions
|
|||
# are met:
|
|||
#
|
|||
# * Redistributions of source code must retain the above copyright
|
|||
# notice, this list of conditions and the following disclaimer.
|
|||
# * Redistributions in binary form must reproduce the above copyright
|
|||
# notice, this list of conditions and the following disclaimer in the
|
|||
# documentation and/or other materials provided with the distribution.
|
|||
# * Neither the name of Matson Systems, Inc. nor the names of its
|
|||
# contributors may be used to endorse or promote products derived
|
|||
# from this software without specific prior written permission.
|
|||
#
|
|||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|||
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|||
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
|
|||
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
|
|||
# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
|
|||
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
|||
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
|||
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
|||
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
|||
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
|
|||
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
|||
# POSSIBILITY OF SUCH DAMAGE.
|
|||
# PAGE_SIZE is the number of rows updated in a single transaction.
|
|||
# This facilitates tables where the number of rows exceeds the systems
|
|||
# memory
|
|||
9d43fc71 | Michael Moll | PAGE_SIZE = 10000
|
|
8a5d3250 | Ohad Levy | ||
namespace :db do
|
|||
namespace :convert do
|
|||
a53137f7 | Ohad Levy | desc 'Convert/import production data to development. DANGER Deletes all data in the development database. Assumes both schemas are already migrated.'
|
|
task :prod2dev => :environment do
|
|||
8a5d3250 | Ohad Levy | # We need unique classes so ActiveRecord can hash different connections
|
|
# We do not want to use the real Model classes because any business
|
|||
# rules will likely get in the way of a database transfer
|
|||
596cf7cf | Timo Goebel | class ProductionModelClass < ActiveRecord::Base # rubocop:disable Rails/ApplicationRecord
|
|
a53137f7 | Ohad Levy | # disable STI
|
|
self.inheritance_column = :_type_disabled
|
|||
8a5d3250 | Ohad Levy | end
|
|
596cf7cf | Timo Goebel | class DevelopmentModelClass < ActiveRecord::Base # rubocop:disable Rails/ApplicationRecord
|
|
a53137f7 | Ohad Levy | # disable STI
|
|
self.inheritance_column = :_type_disabled
|
|||
8a5d3250 | Ohad Levy | end
|
|
ActiveRecord::Base.establish_connection(:production)
|
|||
a53137f7 | Ohad Levy | skip_tables = ["schema_info", "schema_migrations"]
|
|
(ActiveRecord::Base.connection.tables - skip_tables).each do |table_name|
|
|||
time = Time.now
|
|||
8a5d3250 | Ohad Levy | ||
a53137f7 | Ohad Levy | ProductionModelClass.establish_connection(:production)
|
|
9d43fc71 | Michael Moll | ProductionModelClass.table_name = table_name
|
|
a53137f7 | Ohad Levy | ProductionModelClass.reset_column_information
|
|
8a5d3250 | Ohad Levy | DevelopmentModelClass.establish_connection(:development)
|
|
7e6244e4 | hhenkel | # turn off Foreign Key checks for development db - this is per session
|
|
f2cddb6b | Greg Sutcliffe | sql = case DevelopmentModelClass.connection.adapter_name.downcase
|
|
when /^mysql/
|
|||
'SET FOREIGN_KEY_CHECKS=0;'
|
|||
when /^postgresql/
|
|||
"ALTER TABLE #{table_name} DISABLE TRIGGER ALL;"
|
|||
end
|
|||
DevelopmentModelClass.connection.execute(sql) if sql
|
|||
9d43fc71 | Michael Moll | DevelopmentModelClass.table_name = table_name
|
|
8a5d3250 | Ohad Levy | DevelopmentModelClass.reset_column_information
|
|
DevelopmentModelClass.record_timestamps = false
|
|||
f2cddb6b | Greg Sutcliffe | # Handle HABTM tables which don't have an id primary key
|
|
# This *shouldn't* be needed but Rails seems to be picking
|
|||
# up the pkey from other tables in some kind of race condition
|
|||
15f0884e | Dominic Cleal | unless ProductionModelClass.column_names.include?('id')
|
|
DevelopmentModelClass.primary_key = nil
|
|||
ProductionModelClass.primary_key = nil
|
|||
end
|
|||
f2cddb6b | Greg Sutcliffe | ||
8a5d3250 | Ohad Levy | # Page through the data in case the table is too large to fit in RAM
|
|
4ebe38c7 | Ohad Levy | offset = count = 0
|
|
2d8b4fef | Daniel Lobato | print "Converting #{table_name}..."
|
|
STDOUT.flush
|
|||
8a5d3250 | Ohad Levy | # First, delete any old dev data
|
|
DevelopmentModelClass.delete_all
|
|||
ee360810 | Michael Moll | until (models = ProductionModelClass.offset(offset).limit(PAGE_SIZE)).empty?
|
|
8a5d3250 | Ohad Levy | ||
count += models.size
|
|||
offset += PAGE_SIZE
|
|||
# Now, write out the prod data to the dev db
|
|||
DevelopmentModelClass.transaction do
|
|||
models.each do |model|
|
|||
a53137f7 | Ohad Levy | new_model = DevelopmentModelClass.new()
|
|
model.attributes.each do |key, value|
|
|||
new_model[key] = value rescue nil
|
|||
end
|
|||
# don't miss the type attribute when using single-table-inheritance
|
|||
new_model[:type] = model[:type] if model[:type].present?
|
|||
f2cddb6b | Greg Sutcliffe | ||
# Write timestamps for things which haven't had them set
|
|||
# as these columns are DEFAULT NOT NULL
|
|||
15f0884e | Dominic Cleal | new_model[:created_at] ||= time if new_model.attributes.include?('created_at')
|
|
new_model[:updated_at] ||= time if new_model.attributes.include?('updated_at')
|
|||
f2cddb6b | Greg Sutcliffe | ||
017e1049 | Ohad Levy | new_model.save(:validate => false)
|
|
8a5d3250 | Ohad Levy | end
|
|
end
|
|||
end
|
|||
f2cddb6b | Greg Sutcliffe | # turn Foreign Key checks back on, for cleanliness
|
|
sql = case DevelopmentModelClass.connection.adapter_name.downcase
|
|||
when /^mysql/
|
|||
'SET FOREIGN_KEY_CHECKS=1;'
|
|||
when /^postgresql/
|
|||
"ALTER TABLE #{table_name} ENABLE TRIGGER ALL;"
|
|||
end
|
|||
85021506 | Michael Moll | DevelopmentModelClass.connection.execute(sql) if sql.present?
|
|
f2cddb6b | Greg Sutcliffe | ||
a53137f7 | Ohad Levy | print "#{count} records converted in #{Time.now - time} seconds\n"
|
|
8a5d3250 | Ohad Levy | end
|
|
end
|
|||
end
|
|||
end
|