Friday, June 23, 2017

Solaris 11 - Migration of Solaris 10 LDOM oracle database server to Solaris11

Solaris 11 - Migration of Solaris 10 LDOM oracle database server to Solaris11


Prerequisites tasks

For Oracle database on Solaris 11 gold server, install following packages.

# pkg publisher
# pkginfo -l SUNWcsl
# pkg info -r motif
# pkg list | grep -i motif
# pkg install library/motif
# pkg list | grep -i xwplt
# pkg info -r xwplt
# pkg info -r SUNWxwplt
# pkg install compatibility/packages/SUNWxwplt
# pkg install compatibility/ucb


Build LDOM

1. Create domain
# ldm add-domain my_ldm-v01
# ldm list

2. Add CPU/Memory/Network info
a. Add CPU/Mem
# ldm add-vcpu 2 my_ldm-v01
# ldm add-memory 2g my_ldm-v01
# ldm list | grep sol11
b. Add network info. since its IPMP, you need to add two NIC,
Get source LDOM network info, and match it to destination
# ldm list-bindings my_ldm-v01
    vnet0            primary-vsw2@primary 
    vnet1            primary-vsw3@primary 

# ldm add-vnet link-prop=phys-state vnet0 primary-vsw0 <ldom>
# ldm add-vnet linkprop=phys-state vnet0 primary-vsw0 my_ldm-v01
# ldm add-vnet linkprop=phys-state vnet1 primary-vsw1 my_ldm-v01
# ldm list-bindings my_ldm-v01
# ldm list | more

3. Assign Disk space

~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Note from storage team:

From Solaris 11 Goldsrv  SOL-MG-P01 (VM: sol-11-goldsrv    LUN 35/ID:  31D)    present to     
SOL-MG-P02 (VM:  my_ldm_v02-sol_11) LUN 48/ID: 035e



1.  From Compellant Control Center, under SOL-MG-P02, create the following VM Folders:
       my_ldm_v03-sol_11
       my_ldm_v02-sol_11

2.  Create the following LUN COPY (not Replay):
      From Solaris 11 Goldsrv  SOL-MG-P01 (VM: sol-11-goldsrv    LUN 35/ID:  31D)    present to      SOL-MG-P02 (VM:  my_ldm_v03-sol_11)  LUN

47/ID:035d
      From Solaris 11 Goldsrv  SOL-MG-P01 (VM: sol-11-goldsrv    LUN 35/ID:  31D)    present to      SOL-MG-P02 (VM:  my_ldm_v02-sol_11) LUN

48/ID: 035e

~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~


a. Scan the LUNS
# cfgadm -al
# cfgadm -al -o show_FCP_dev
b. Find LUN Number
# luxadm display /dev/rdsk/c3t6000D310006D640000000000000001FFd0s2
Look under    
Device Address              5000d310006d6431,1c
here, for eg, 1c is the hex number, you have to conver it to DEC, use programmer calculator to change.
to convert 1c to dec it is 28.

# echo | format | egrep "0356|0357"
# ldm add-vdsdev /dev/dsk/c0t6000D310006D64000000000000000356d0s2 DL42@primary-vds0
# ldm add-vdsdev /dev/dsk/c0t6000D310006D64000000000000000357d0s2 DL43@primary-vds0
# ldm add-vdisk VDL42 DL42@primary-vds0 my_ldm_v04_sol11
# ldm add-vdisk VDL43 DL43@primary-vds0 my_ldm-v01
# ldm list-bindings my_ldm_v04_sol11
# ldm list-bindings my_ldm-v01
# ldm list-bindings my_ldm_v04_sol11
# ldm list

4. Set auto boot to false so that system does not boot upon power on.
# ldm set-variable auto-boot\?=false my_ldm-v01
# ldm set-variable auto-boot\?=false my_ldm_v04_sol11

5. Bind and start LDOM
# ldm list
# ldm bind my_ldm_v04_sol11
# ldm start my_ldm_v04_sol11
LDom my_ldm_v04_sol11 started
# ldm start my_ldm-v01


6. List detail about Solaris 11 LDOM
# ldm list-bindings my_ldm-v01 | more

7. List detail about SOlaris 10 server LDOM
# ldm list-bindings my_ldm_v05 | more

8. Connect to the console
# telnet 0 5013
{0} ok boot -s

Enter user name for system maintenance (control-d to bypass):

root@sol-11-goldsrv:~# ifconfig -a
ipmp0: flags=10c001000842<BROADCAST,RUNNING,MULTICAST,IPv4,DUPLICATE,IPMP,PHYSRUNNING> mtu 1500 index 2
        inet 192.168.10.66 netmask ffffff80 broadcast 192.168.10.127
        groupname ipmp0
net0: flags=100001000843<UP,BROADCAST,RUNNING,MULTICAST,IPv4,PHYSRUNNING> mtu 1500 index 3
net1: flags=100061000843<UP,BROADCAST,RUNNING,MULTICAST,IPv4,STANDBY,INACTIVE,PHYSRUNNING> mtu 1500 index 4


9. change the following info based on your new env for initial set up of ip/hostname and run it
root@sol-11-goldsrv:/var/tmp/initial-setup# vi setup.sh

ENCLAVE=<dev>
HOSTNAME=<sol-11-goldsrv>
DOMAIN=<expanor.dev>
IP_ADDRESS=<192.168.10.69>

root@sol-11-goldsrv:/var/tmp/initial-setup# export EDITOR=vi
root@sol-11-goldsrv:/var/tmp/initial-setup# export TERM=vt100
root@sol-11-goldsrv:/var/tmp/initial-setup# sh setup.sh


root@sol-11-goldsrv:/var/tmp/initial-setup# svcs -a | grep network/physical
online          9:53:04 svc:/network/physical:upgrade
online          9:54:13 svc:/network/physical:default


root@sol-11-goldsrv:/var/tmp/CAC# ifconfig -a

ipmp0: flags=108001000843<UP,BROADCAST,RUNNING,MULTICAST,IPv4,IPMP,PHYSRUNNING> mtu 1500 index 2
        inet 192.168.10.69 netmask ffffff80 broadcast 192.168.10.127
        groupname ipmp0
net0: flags=100001000843<UP,BROADCAST,RUNNING,MULTICAST,IPv4,PHYSRUNNING> mtu 1500 index 3
net1: flags=100061000843<UP,BROADCAST,RUNNING,MULTICAST,IPv4,STANDBY,INACTIVE,PHYSRUNNING> mtu 1500 index 4

Verify hostname/ipaddress of the output.
root@sol-11-goldsrv:/var/tmp/CAC/DEV# svcprop identity:node | more


10. Change role into user.

root@sol-11-goldsrv:/var/tmp/CAC/DEV# usermod -R "" jay
root@sol-11-goldsrv:/var/tmp/CAC/DEV# usermod -R "" sam
root@sol-11-goldsrv:/var/tmp/CAC/DEV# rolemod -K type=normal root


11. Now, reboot the machine and login to server using new IP and set up LDAP-CAC
my_ldm_v04 console login:
root@sol-11-goldsrv:/var/tmp/CAC/DEV# reboot
root@sol-11-goldsrv:/# cd /var/tmp/CACDEV/
root@sol-11-goldsrv:/var/tmp/CAC/DEV# sh ldap_inst_dev.sh



12. Go to source server, review cat /etc/ passwd, shadow and group and copy all the user entry related to database/application account.

# egrep "gridmon|iascan|oinstall|oracle" /etc/passwd
# egrep "gridmon|iascan|oinstall|oracle" /etc/shadow
# for i in gridmon iascan oinstall oracle;  do id -a $i; done
# egrep "102|501|503" /etc/group


13. Copy the content of /export/home directory and also /var/opt/oracle
a. Enable direct root login to destination host
# vi /usr/local/ssh6.6p1/etc/sshd_config
PermitRootLogin Yes

b. And restart the service
# /etc/init.d/ssh stop/start



14. Copy important files/directories (/home dir /var/opt/oracle) from source, and update to destination server.
A. Copy from source server
a. Copy directly (use tar and scp without saving on source host

# cd /var/opt
# tar cfE - oracle | ssh root@192.168.10.69 "( cd /var/opt; tar xfBp - )"
# cd /export/home
# tar cfE - gridmon oradev oinstall | ssh root@192.168.10.69 "( cd /export/home; tar xfBp - )"


b. Tar the content at /var/tmp and use scp to copy
# cd /export/home/; du -sh gridmon iascan oinstall oracle
# tar -cvf /var/tmp/home_dir.tar gridmon iascan oinstall oracle
# cd /var/opt;# du -sh *
# tar -cvf /var/tmp/var_opt_oracle.tar oracle
# ls -lh /var/tmp/var_opt_oracle.tar /var/tmp/home_dir.tar
# cd /var/tmp; scp var_opt_oracle.tar home_dir.tar sam@192.168.10.69:/var/tmp


B. Update destination server if needed

root@my_ldm_v04:~# vi /etc/passwd
root@my_ldm_v04:~# vi /etc/passwd
root@my_ldm_v04:~# vi /etc/shadow
root@my_ldm_v04:~#
root@my_ldm_v04:~# vi /etc/group
root@my_ldm_v04:~#

# tar -tvf /var/tmp/home_dir.tar | more
# cd /export/home/; tar -xvf /var/tmp/home_dir.tar
# cd /var/opt; tar -xvf /var/tmp/var_opt_oracle.tar
# cd oracle; ls
# df -h

verify user env,
# su - oinstall
[oinstall@my_ldm_v04]>env


15. Update sudoers file
# visudo

make sure you verify source/destination file content.


16. Update vfstab, system, and projects file
# vi /etc/vfstab
make sure you verify source/destination file content.



root@my_ldm_v04:~# cat /etc/system

Review/change your zfs arc size.

set zfs:zfs_arc_max=2147483648
set zfs:zfs_arc_min=67108864
* Setting max file descriptors
set rlim_fd_max=65536
set rlim_fd_cur=1024
* To prevent buffer overflow
* set noexec_user_stack = 1
* set noexec_user_stack_log = 1
* Enable BSM Auditing
set c2audit:audit_load = 1
* Monitor for unauthorized connections of mobile devices
exclude: scsa2usb


Review oracle sga size and update your project file. (GB in size?)
# projects -l

user.oinstall
        projid : 101
        comment: ""
        users  : oinstall
        groups : oinstall
                 dba
        attribs: process.max-sem-nsems=(priv,810,deny)
                 project.max-sem-ids=(priv,512,deny)
                 project.max-shm-ids=(priv,512,deny)
                 project.max-shm-memory=(priv,4294967296,deny)
oracle
        projid : 220
        comment: ""
        users  : oinstall
                 oracle
        groups : oinstall
                 dba
        attribs: process.max-sem-nsems=(priv,256,deny)
                 project.max-sem-ids=(priv,100,deny)
                 project.max-shm-ids=(priv,100,deny)
                 project.max-shm-memory=(priv,4294967296,deny)
root@my_ldm_v04:~#



17. refresh nfs/client
# svcadm enable -r nfs/client


18. Now, time to shutdown source system. stop the source LDOM.
Login to console on destination host and change the IP address info

root@SOL-MG-P02 # ldm stop my_ldm_v05
root@SOL-MG-P02 # ldm stop my_ldm_v04
root@SOL-MG-P02 # ldm unbind my_ldm_v05
root@SOL-MG-P02 # ldm unbind my_ldm_v04

# telnet 0 5014
a. Review existing IP/host info
# ifconfig -a | more
ipmp0: flags=108001000843<UP,BROADCAST,RUNNING,MULTICAST,IPv4,IPMP,PHYSRUNNING> mtu 1500 index 2
        inet 192.168.10.69 netmask ffffff80 broadcast 192.168.10.127
        groupname ipmp0
net0: flags=100001000843<UP,BROADCAST,RUNNING,MULTICAST,IPv4,PHYSRUNNING> mtu 1500 index 3
net1: flags=100061000843<UP,BROADCAST,RUNNING,MULTICAST,IPv4,STANDBY,INACTIVE,PHYSRUNNING> mtu 1500 index 4

# ipadm show-addr
b. Remove old IPMP setting
# ipadm delete-addr ipmp0/v4

c. Assign new IP
# ipadm create-addr -T static -a  192.168.10.33 ipmp0
# svccfg -s identity:node setprop private_data/mapped_addresses=192.168.10.33
# svcadm refresh identity:node
# svcadm restart identity:node
# svcprop identity:node | more

19. Login to control domain and set CPU/MEM
Note: change CPU/Mem while on multi user mode or on power off state.

a. Change CPU/Mem info
root@SOL-MG-P02 # ldm set-vcpu 16 my_ldm_v04_sol11
root@SOL-MG-P02 # ldm set-mem 10G my_ldm_v04_sol11

root@SOL-MG-P02 # ldm set-mem 8G my_ldm-v01
root@SOL-MG-P02 # ldm set-vcpu 16 my_ldm-v01

b. if you are at OK prompt, stop the LDOM and allocate resources.
root@SOL-MG-P02 # ldm stop my_ldm-v01
root@SOL-MG-P02 # ldm set-mem 8G my_ldm-v01


root@SOL-MG-P02 # ldm set-mem 8G my_ldm-v01
root@SOL-MG-P02 # ldm set-vcpu 16 my_ldm-v01
root@SOL-MG-P02 #



20. If you are on multiuser mode, reboot your system.
# init 5 or reboot

or

if your VM is powered off, Power on your VM and boot the system and login to console

root@SOL-MG-P02 # ldm start my_ldm-v01
root@SOL-MG-P02 # ldm start my_ldm-v01

Login to console
# telnet 0 5014

{0} ok boot


Once you login, check logs, filesystem and login using CAC

root@SOL-MG-P02 # ldm list
NAME             STATE      FLAGS   CONS    VCPU  MEMORY   UTIL  UPTIME
primary          active     -n-cv-  UART    8     8G       1.2%  48d 23h 24m
my_ldm-v01        bound      ------  5013    16    8G
my_ldm_v04_sol11   active     -n----  5014    16    10G      0.1%  1d 7h 3m
my_ldm_v02       active     -n----  5010    32    16G      0.2%  48d 23h 19m
my_ldm_v03       active     -n----  5001    16    8G       1.7%  48d 23h 19m
my_ldm_v05       inactive   ------          16    8G
my_ldm_v04       inactive   ------          16    10


To change IP Address:

# Delete OLD IP Address
echo "Deleting old IP Address" && sleep 1; IPADDROBJ=`ipadm show-addr | sed -n '/^ipmp*/p' | awk '{print $1}'` ipadm delete-addr "$IPADDROBJ"

# Assigning NEW IP Address
echo "Assigning New IP Address" && sleep 1; ipadm create-addr -T static -a "$IP_ADDRESS" ipmp0

Update IP Address in SMF:
svccfg -s identity:node setprop private_data/mapped_addresses=192.168.10.41
svcadm refresh identity:node
svcadm restart identity:node

**don't forget to copy over /var/opt/oracle

**don't forget NFS entries
SOL-MG-P01:/repository - /repository nfs - yes rw,soft 192.168.10.16:/backup - /BACKUP nfs - yes

hard,rw,rsize=32768,wsize=32768,suid,proto=tcp,vers=3

** don't forget to update /etc/hosts,  add #repo entry as well # Repo
192.168.10.11   SOL-MG-P01.expanor.dev    SOL-MG-P01




root@my_ldm_v04:/data#  svcs -a | grep rad
disabled       20:11:26 svc:/system/device/mpxio-upgrade:default
disabled       20:11:29 svc:/network/inetd-upgrade:default
disabled       20:11:29 svc:/system/device/policy-upgrade:default
disabled       20:11:36 svc:/system/rad:remote
online         20:11:30 svc:/network/connectx/unified-driver-post-upgrade:default
online         20:11:31 svc:/system/name-service/upgrade:default
online         20:11:36 svc:/network/eoib/eoib-post-upgrade:default
online         20:11:44 svc:/system/logadm-upgrade:default
online         20:11:45 svc:/system/rad:local
online         20:11:45 svc:/system/rad:local-http
online         20:11:46 svc:/network/physical:upgrade
online         20:11:46 svc:/network/location:upgrade
root@my_ldm_v04:/data# svcadm disable svc:/system/rad:local
root@my_ldm_v04:/data# svcadm disable svc:/system/rad:local-http
root@my_ldm_v04:/data#


root@my_ldm_v04:~#  svcs -a | grep -i rad
disabled       Jun_19   svc:/system/device/mpxio-upgrade:default
disabled       Jun_19   svc:/network/inetd-upgrade:default
disabled       Jun_19   svc:/system/device/policy-upgrade:default
disabled       Jun_19   svc:/system/rad:remote
online         Jun_19   svc:/network/connectx/unified-driver-post-upgrade:default
online         Jun_19   svc:/system/name-service/upgrade:default
online         Jun_19   svc:/network/eoib/eoib-post-upgrade:default
online         Jun_19   svc:/system/rad:local-http
online         Jun_19   svc:/system/rad:local
online         Jun_19   svc:/system/logadm-upgrade:default
online         Jun_19   svc:/network/physical:upgrade
online         Jun_19   svc:/network/location:upgrade
root@my_ldm_v04:~#



#  passwd root
#  passwd sam
#  chmod 700 /export/home/*
#  for i in `ls /export/home/*/.bash_history`; do ls -l $i; done
#  for i in `ls /export/home/*/.bash_history`; do cat /dev/null > $i; done
#  for i in `ls /export/home/*/.bash_history`; do ls -l $i; done
#  for i in `ls /export/home/*/.profile /export/home/*/.bash_profile`; do echo "export HISTCONTROL=ignorespace" >> $i; done
#  for i in `ls /export/home/*/.profile /export/home/*/.bash_profile`; do grep HISTCONTROL $i; done
#  ls -ld /export/home/*
#  svcs -a | grep -i mile
#  uname -a

No comments:

Post a Comment