#!/usr/bin/python

# -*- coding: utf-8 -*-

# Licensed to the Apache Software Foundation (ASF) under one

# or more contributor license agreements. See the NOTICE file

# distributed with this work for additional information

# regarding copyright ownership. The ASF licenses this file

# to you under the Apache License, Version 2.0 (the

# "License"); you may not use this file except in compliance

# with the License. You may obtain a copy of the License at

#

# http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing,

# software distributed under the License is distributed on an

# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY

# KIND, either express or implied. See the License for the

# specific language governing permissions and limitations

# under the License.

import os

import sys

import subprocess

import glob

from random import choice

import string

from optparse import OptionParser

import commands

import shutil

import socket

# squelch mysqldb spurious warnings

import warnings

warnings.simplefilter('ignore')

# ---- This snippet of code adds the sources path and the waf configured PYTHONDIR to the Python path ----

# ---- We do this so cloud_utils can be looked up in the following order:

# ---- 1) Sources directory

# ---- 2) waf configured PYTHONDIR

# ---- 3) System Python path

for pythonpath in (

"@PYTHONDIR@",

os.path.join(os.path.dirname(__file__),os.path.pardir,os.path.pardir,"python","lib"),

):

if os.path.isdir(pythonpath): sys.path.insert(0,pythonpath)

# ---- End snippet of code ----

from cloud_utils import check_selinux, CheckFailed, resolves_to_ipv6

def runCmd(cmds):

process = subprocess.Popen(' '.join(cmds), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

stdout, stderr = process.communicate()

if process.returncode != 0:

raise Exception(stderr)

return stdout

class DBDeployer(object):

parser = None

options = None

args = None

serversetup = None

ip = None

user,password,host,port,rootuser,rootpassword = [None,None,None,None,None,None]

isDebug = False

mgmtsecretkey = None

dbsecretkey = None

encryptiontype = None

dbConfPath = r"@MSCONF@"

dbFilesPath = r"@SETUPDATADIR@"

dbDotProperties = {}

dbDotPropertiesIndex = 0

encryptionKeyFile = '@MSCONF@/key'

encryptionJarPath = '@COMMONLIBDIR@/lib/jasypt-1.9.2.jar'

success = False

magicString = 'This_is_a_magic_string_i_think_no_one_will_duplicate'

tmpMysqlFile = os.path.join(os.path.expanduser('~/'), 'cloudstackmysql.tmp.sql')

mysqlBinPath = None

def preRun(self):

def backUpDbDotProperties():

dbpPath = os.path.join(self.dbConfPath, 'db.properties')

copyPath = os.path.join(self.dbConfPath, 'db.properties.origin')

if os.path.isfile(dbpPath):

shutil.copy2(dbpPath, copyPath)

backUpDbDotProperties()

def postRun(self):

def cleanOrRecoverDbDotProperties():

dbpPath = os.path.join(self.dbConfPath, 'db.properties')

copyPath = os.path.join(self.dbConfPath, 'db.properties.origin')

if os.path.isfile(copyPath):

if not self.success:

shutil.copy2(copyPath, dbpPath)

os.remove(copyPath)

cleanOrRecoverDbDotProperties()

if os.path.exists(self.tmpMysqlFile):

os.remove(self.tmpMysqlFile)

def info(self, msg, result=None):

output = ""

if msg is not None:

output = "%-80s"%msg

if result is True:

output += "[ \033[92m%-2s\033[0m ]\n"%"OK"

elif result is False:

output += "[ \033[91m%-6s\033[0m ]\n"%"FAILED"

sys.stdout.write(output)

sys.stdout.flush()

def debug(self, msg):

msg = "DEBUG:%s"%msg

sys.stdout.write(msg)

sys.stdout.flush()

def putDbProperty(self, key, value):

if self.dbDotProperties.has_key(key):

(oldValue, index) = self.dbDotProperties[key]

self.dbDotProperties[key] = (value, index)

else:

self.dbDotProperties[key] = (value, self.dbDotPropertiesIndex)

self.dbDotPropertiesIndex += 1

def getDbProperty(self, key):

if not self.dbDotProperties.has_key(key):

return None

(value, index) = self.dbDotProperties[key]

return value

def runMysql(self, text, table, isRoot=False):

kwargs = {}

if not isRoot:

kwargs['user'] = self.user

if self.password != '': kwargs['passwd'] = self.password

else:

kwargs['user'] = self.rootuser

if self.rootpassword != '': kwargs['passwd'] = self.rootpassword

kwargs['port'] = self.port

kwargs['host'] = self.host

if self.mysqlBinPath is not None:

mysqlPath = '"' + self.mysqlBinPath + os.sep + "mysql" + '"'

else:

mysqlPath = "mysql"

try:

mysqlCmds = [mysqlPath, '--user=%s'%kwargs['user'], '--host=%s'%kwargs['host'], '--port=%s'%kwargs['port']]

if kwargs.has_key('passwd'):

mysqlCmds.append('--password=%s'%kwargs['passwd'])

file(self.tmpMysqlFile, 'w').write(text)

mysqlCmds.append('<')

mysqlCmds.append(self.tmpMysqlFile)

runCmd(mysqlCmds)

except Exception, e:

err = '''Encountering an error when executing mysql script

----------------------------------------------------------------------

table:

%s

Error:

%s

Sql parameters:

%s

----------------------------------------------------------------------

'''%(table, e.__str__(), kwargs)

self.errorAndExit(err)

def errorAndContinue(self, msg):

sys.stderr.write(msg)

sys.stderr.flush()

def errorAndExit(self, msg):

self.postRun()

err = '''\n\nWe apologize for below error:

***************************************************************

%s

***************************************************************

Please run:

cloud-setup-database -h

for full help

''' % msg

sys.stderr.write(err)

sys.stderr.flush()

sys.exit(1)

def setupDBSchema(self):

if not self.rootuser:

self.info("No mysql root user specified, will not create Cloud DB schema\n", None)

return

replacements = (

("CREATE USER cloud identified by 'cloud';",

"CREATE USER %s@`localhost` identified by '%s'; CREATE USER %s@`%%` identified by '%s';"%(

(self.user,self.password,self.user,self.password)

)),

("cloud identified by 'cloud';",

"%s identified by '%s';"%(self.user,self.password)),

("cloud@`localhost` identified by 'cloud'",

"%s@`localhost` identified by '%s'"%(self.user,self.password)),

("cloud@`%` identified by 'cloud'",

"%s@`%%` identified by '%s'"%(self.user,self.password)),

("to cloud@`localhost`",

"to %s@`localhost`"%self.user),

("to cloud@`%`",

"to %s@`%%`"%self.user),

("TO cloud@`localhost`",

"to %s@`localhost`"%self.user),

("TO cloud@`%`",

"to %s@`%%`"%self.user),

("WHERE `User` = 'cloud' and host =",

"WHERE `User` = '%s' and host ="%self.user),

("DROP USER 'cloud'",

"DROP USER '%s'"%self.user),

("CALL `test`.`drop_user_if_exists`() ;",

""),

)

for f in ["create-database","create-schema", "create-database-premium","create-schema-premium"]:

p = os.path.join(self.dbFilesPath,"%s.sql"%f)

if not os.path.exists(p): continue

text = file(p).read()

for t, r in replacements: text = text.replace(t,r)

self.info("Applying %s"%p)

self.runMysql(text, p, True)

self.info(None, True)

if self.serversetup:

conf = os.path.join(self.dbConfPath, 'db.properties')

pcp = os.path.pathsep.join( glob.glob( os.path.join ( r"@PREMIUMJAVADIR@" , "*" ) ) )

systemjars = r"@SYSTEMJARS@".split()

try:

output = runCmd(['build-classpath'] + systemjars)

systemcp = output.strip()

except Exception, e:

systemcp = r"@SYSTEMCLASSPATH@"

mscp = r"@MSCLASSPATH@"

depscp = r"@DEPSCLASSPATH@"

classpath = os.path.pathsep.join([pcp,systemcp,depscp,mscp,conf])

try:

runCmd(["java","-cp",classpath,"com.cloud.test.DatabaseConfig", self.serversetup])

except Exception, e:

self.errorAndExit("Apply %s failed"%self.serversetup)

else:

p = os.path.join(self.dbFilesPath, 'server-setup.sql')

text = file(p).read()

self.info("Applying %s"%p)

self.runMysql(text, p, True)

self.info(None, True)

for f in ["templates"]:

p = os.path.join(self.dbFilesPath,"%s.sql"%f)

text = file(p).read()

self.info("Applying %s"%p)

self.runMysql(text, p, True)

self.info(None, True)

p = os.path.join(self.dbFilesPath,"schema-level.sql")

if os.path.isfile(p):

text = file(p).read()

self.info("Applying %s"%p)

self.runMysql(text, p, True)

self.info(None, True)

def prepareDBFiles(self):

def prepareDBDotProperties():

dbpPath = os.path.join(self.dbConfPath, 'db.properties')

dbproperties = file(dbpPath).read().splitlines()

newdbp = []

emptyLine = 0

for line in dbproperties:

passed = False

line = line.strip()

if line.startswith("#"): key = line; value = ''; passed = True

if line == '' or line == '\n': key = self.magicString + str(emptyLine); value = ''; emptyLine += 1; passed = True

try:

if not passed:

(key, value) = line.split('=', 1)

if key == "cluster.node.IP": value = self.ip

if key == "db.cloud.username": value = self.user

if key == "db.cloud.password": value = self.password

if key == "db.cloud.host": value = self.host

if key == "db.cloud.port": value = self.port

if key == "db.usage.username": value = self.user

if key == "db.usage.password": value = self.password

if key == "db.usage.host": value = self.host

if key == "db.usage.port": value = self.port

except Exception, e:

err = '''Wrong format in %s (%s):

Besides comments beginning "#" and empty line, all key-value pairs must be in formula of

key=value

for example:

db.cloud.username = cloud

''' % (dbpPath, line)

self.errorAndExit(err)

self.putDbProperty(key, value)

self.info("Preparing %s"%dbpPath, True)

self.putDbProperty("region.id", self.options.regionid)

prepareDBDotProperties()

def finalize(self):

def finalizeDbProperties():

entries = []

for key in self.dbDotProperties.keys():

(value, index) = self.dbDotProperties[key]

if key.startswith("#"):

entries.insert(index, key)

elif key.startswith(self.magicString):

entries.insert(index, '')

else:

entries.insert(index, "%s=%s"%(key, value))

file(os.path.join(self.dbConfPath, 'db.properties'), 'w').write('\n'.join(entries))

self.info("Finalizing setup ...", None)

finalizeDbProperties()

self.info(None, True)

self.success = True # At here, we have done successfully and nothing more after this flag is set

def grabSystemInfo(self):

def getIpAddr():

try:

ip = socket.gethostbyname(socket.gethostname())

return ip

except Exception, e:

return "127.0.0.1"

if not self.ip:

self.ip = getIpAddr()

self.info("Detected local IP address as %s, will use as cluster management server node IP" % self.ip, True)

def checkSystemSetup(self):

def checkCloudDbFiles():

self.info("Checking Cloud database files ...", None)

dbfpaths = [ os.path.join(self.dbConfPath,"db.properties") ]

for dbf in dbfpaths:

if not os.path.exists(dbf):

self.errorAndExit("Cannot find %s"%dbf)

coreSchemas = ['create-database.sql', 'create-schema.sql', 'templates.sql']

if not self.serversetup:

coreSchemas.append('server-setup.sql')

checkingList = [os.path.join(self.dbFilesPath, x) for x in coreSchemas]

checkingList.append(self.encryptionJarPath)

for f in checkingList:

if not os.path.isfile(f):

self.errorAndExit("Cloud DB required file %s was not found"%f)

self.info(None, True)

def checkDbserverHostname():

self.info("Checking mysql server hostname ...", None)

if resolves_to_ipv6(self, self.port):

err = "%s resolves to an IPv6 address. The CloudStack does not support IPv6 yet.\nPlease fix this issue in either /etc/hosts or your DNS configuration.\n"%self.host

self.errorAndExit(err)

self.info(None, True)

def checkHostName():

self.info("Checking local machine hostname ...", None)

try:

output= socket.getfqdn()

except Exception, e:

err = "The host name of this computer does not resolve to an IP address.\nPlease use your operating system's network setup tools to fix this ('hostname --fqdn' %s).\n"%e.__str__()

self.errorAndExit(err)

self.info(None, True)

def checkSELinux():

self.info("Checking SELinux setup ...", None)

try:

check_selinux()

except CheckFailed, e:

self.info("checkSelinux failed with error continuing...", None)

self.errorAndContinue(e.__str__())

except OSError, e:

if e.errno == 2: pass

else:

self.info("checkSelinux failed with error continuing...", None)

self.errorAndContinue(e.__str__())

self.info(None, True)

checkCloudDbFiles()

checkHostName()

checkSELinux()

def processEncryptionStuff(self):

def encrypt(input):

cmd = ['java','-classpath','"' + self.encryptionJarPath + '"','org.jasypt.intf.cli.JasyptPBEStringEncryptionCLI', 'encrypt.sh', 'input=%s'%input, 'password=%s'%self.mgmtsecretkey,'verbose=false']

return runCmd(cmd).strip('\r\n')

def saveMgmtServerSecretKey():

if self.encryptiontype == 'file':

file(self.encryptionKeyFile, 'w').write(self.mgmtsecretkey)

def formatEncryptResult(value):

return 'ENC(%s)'%value

def encryptDBSecretKey():

self.putDbProperty('db.cloud.encrypt.secret', formatEncryptResult(encrypt(self.dbsecretkey)))

def encryptDBPassword():

dbPassword = self.getDbProperty('db.cloud.password')

if dbPassword == '': return # Don't encrypt empty password

if dbPassword == None: self.errorAndExit('Cannot find db.cloud.password in %s'%os.path.join(self.dbConfPath, 'db.properties'))

self.putDbProperty('db.cloud.password', formatEncryptResult(encrypt(dbPassword)))

usagePassword = self.getDbProperty('db.usage.password')

if usagePassword == '': return # Don't encrypt empty password

if usagePassword == None: self.errorAndExit('Cannot find db.usage.password in %s'%os.path.join(self.dbConfPath, 'db.properties'))

self.putDbProperty('db.usage.password', formatEncryptResult(encrypt(usagePassword)))

self.info("Processing encryption ...", None)

self.putDbProperty("db.cloud.encryption.type", self.encryptiontype)

saveMgmtServerSecretKey()

encryptDBSecretKey()

encryptDBPassword()

self.info(None, True)

def parseOptions(self):

def parseOtherOptions():

if self.options.rootcreds:

self.rootuser,self.rootpassword = parseUserAndPassword(self.options.rootcreds)

if self.rootuser == self.user:

self.errorAndExit("--deploy-as= user name cannot be the user name supplied for the connection credentials")

self.info("Mysql root user name:%s"%self.rootuser, True)

self.info("Mysql root user password:%s"%"******", True)

if self.options.serversetup:

if not self.options.rootcreds:

self.errorAndExit("--auto= requires valid --deploy-as= credentials")

if os.path.isfile(self.options.serversetup):

self.errorAndExit("%s is not a valid file"%self.options.serversetup)

self.serversetup = self.options.serversetup

self.info("User specified server-setup.sql file at %s"%self.serversetup, True)

if self.options.mshostip:

self.ip = self.options.mshostip

self.info("Using specified cluster management server node IP %s" % self.options.mshostip, True)

self.encryptiontype = self.options.encryptiontype

self.mgmtsecretkey = self.options.mgmtsecretkey

self.dbsecretkey = self.options.dbsecretkey

self.isDebug = self.options.debug

if self.options.dbConfPath:

self.dbConfPath = self.options.dbConfPath

if self.options.dbFilesPath:

self.dbFilesPath = self.options.dbFilesPath

if self.options.encryptionKeyFile:

self.encryptionKeyFile = self.options.encryptionKeyFile

if self.options.encryptionJarPath:

self.encryptionJarPath = self.options.encryptionJarPath

if self.options.mysqlbinpath:

self.mysqlBinPath = self.options.mysqlbinpath

def parseUserAndPassword(cred):

stuff = cred.split(':')

if len(stuff) != 1 and len(stuff) != 2:

self.errorAndExit("Invalid user name and password format, must be in format of user:password (%s)"%cred)

user = stuff[0]

if len(user) < 1:

self.errorAndExit("Invalid user name and password format, must be in format of user:password, user name can not be empty")

if len(stuff) == 1:

password = ''

else:

password = stuff[1]

forbidden = "' \\`"

for f in forbidden:

if f in user: self.errorAndExit("User name cannot have the %r characters"%f)

if f in password: self.errorAndExit("Password cannot have the %r characters"%f)

return user, password

def parseCasualCredit():

def parseHostInfo(info):

stuff = info.split(":")

if len(stuff) == 1:

host = stuff[0]

port = 3306

elif len(stuff) == 2:

host = stuff[0]

try: port = int(stuff[1])

except ValueError: self.errorAndExit("The database port must be an integer (%s)"%stuff[1])

if port < 1: self.errorAndExit("The database port must be a positive integer (%s)"%stuff[1])

else:

self.errorAndExit("Invalid host and port format, it must be in format of host:port (%s)"%info)

return host, port

if len(self.args) == 0:

self.errorAndExit("Please specify user:password@hostname")

if len(self.args) > 1:

self.errorAndExit("There are more than one parameters for user:password@hostname (%s)"%self.args)

arg = self.args[0]

try:

try:

splitIndex = arg.rindex('@')

except ValueError:

# If it failed to find @, use host=localhost

splitIndex = len(arg)

arg += "@localhost"

finally:

stuff = [arg[:splitIndex], arg[splitIndex+1:]]

self.user,self.password = parseUserAndPassword(stuff[0])

self.host,self.port = parseHostInfo(stuff[1])

self.info("Mysql user name:%s"%self.user, True)

self.info("Mysql user password:%s"%"******", True)

self.info("Mysql server ip:%s"%self.host, True)

self.info("Mysql server port:%s"%self.port, True)

def validateParameters():

if self.encryptiontype != 'file' and self.encryptiontype != 'web':

self.errorAndExit('Wrong encryption type %s, --encrypt-type can only be "file" or "web'%self.encryptiontype)

#---------------------- option parsing and command line checks ------------------------

usage = """%prog user:[password]@mysqlhost:[port] [--deploy-as=rootuser:[rootpassword]] [--auto=/path/to/server-setup.xml] [-e ENCRYPTIONTYPE] [-m MGMTSECRETKEY] [-k DBSECRETKEY] [--debug]

This command sets up the CloudStack Management Server and CloudStack Usage Server database configuration (connection credentials and host information) based on the first argument.

If the the --deploy-as option is present, this command will also connect to the database using the administrative credentials specified as the value for the --deploy-as argument, construct the database environment needed to run the CloudStack Management Server, and alter the password specified for the user in the first argument. In this case, the user name specified in --deploy-as= cannot be the same as the user name specified for the connection credentials that the CloudStack Management Server will be set up with.

If a server-setup.xml cloud setup information file is specified with the --auto option, this command will also construct a customized database environment according to the cloud setup information in the file.

The port and the password are optional and can be left out.. If host is omitted altogether, it will default to localhost.

Examples:

%prog cloud:secret

sets user cloud and password 'secret' up in

@MSCONF@/db.properties, using localhost as the

database server

%prog sheng:rules@192.168.1.1

sets these credentials up in @MSCONF@/db.properties

%prog alex:founder@1.2.3.4 --deploy-as=root:nonsense

sets alex up as the MySQL user, then connects as the root user

with password 'nonsense', and recreates the databases, creating

the user alex with password 'founder' as necessary

%prog alex:founder@1.2.3.4 --deploy-as=root:nonsense -e file -m password -k dbpassword

In addition actions performing in above example, using 'password' as management server encryption key

and 'dbpassword' as database encryption key, saving management server encryption key to a file as the

encryption type specified by -e is file.

%prog alena:tests@5.6.7.8 --deploy-as=root:nonsense --auto=/root/server-setup.xml

sets alena up as the MySQL user, then connects as the root user

with password 'nonsense' to server 5.6.7.8, then recreates the

databases and sets up the alena user, then performs an automated

database setup using the information in server-setup.xml

"""

self.parser = OptionParser(usage=usage)

self.parser.add_option("-v", "--debug", action="store_true", dest="debug", default=False,

help="If enabled, print the commands it will run as they run")

self.parser.add_option("-d", "--deploy-as", action="store", type="string", dest="rootcreds", default="",

help="Colon-separated user name and password of a MySQL user with administrative privileges")

self.parser.add_option("-a", "--auto", action="store", type="string", dest="serversetup", default="",

help="Path to an XML file describing an automated unattended cloud setup")

self.parser.add_option("-e", "--encrypt-type", action="store", type="string", dest="encryptiontype", default="file",

help="Encryption method used for db password encryption. Valid values are file, web. Default is file.")

self.parser.add_option("-m", "--managementserver-secretkey", action="store", type="string", dest="mgmtsecretkey", default="password",

help="Secret key used to encrypt confidential parameters in db.properties. A string, default is password")

self.parser.add_option("-k", "--database-secretkey", action="store", type="string", dest="dbsecretkey", default="password",

help="Secret key used to encrypt sensitive database values. A string, default is password")

self.parser.add_option("-i", "--mshost", action="store", type="string", dest="mshostip", default="",

help="Cluster management server host IP. A string, by default it will try to detect a local IP")

self.parser.add_option("-r", "--regionid", action="store", type="string", dest="regionid", default="1",

help="Region Id for the management server cluster")

self.parser.add_option("-c", "--db-conf-path", action="store", dest="dbConfPath", help="The path to find db.properties which hold db properties")

self.parser.add_option("-f", "--db-files-path", action="store", dest="dbFilesPath", help="The path to find sql files to create initial database(s)")

self.parser.add_option("-j", "--encryption-jar-path", action="store", dest="encryptionJarPath", help="The path to the jasypt library to be used to encrypt the values in db.properties")

self.parser.add_option("-n", "--encryption-key-file", action="store", dest="encryptionKeyFile", help="The name of the file in which encryption key to be generated")

self.parser.add_option("-b", "--mysql-bin-path", action="store", dest="mysqlbinpath", help="The mysql installed bin path")

(self.options, self.args) = self.parser.parse_args()

parseCasualCredit()

parseOtherOptions()

validateParameters()

def run(self):

try:

self.preRun()

self.parseOptions()

self.checkSystemSetup()

self.grabSystemInfo()

self.prepareDBFiles()

self.setupDBSchema()

self.processEncryptionStuff()

self.finalize()

finally:

self.postRun()

print ''

print "CloudStack has successfully initialized database, you can check your database configuration in %s"%os.path.join(self.dbConfPath, 'db.properties')

print ''

if __name__ == "__main__":

o = DBDeployer()

o.run()