Openstack Swift構築太郎

openstack swiftの構築手順。 OSはubuntu 10.04 swiftのverは1.7.0(folsum) 構成は、 LB配下にproxy:2 account:3 container:3 object:3 公式ドキュメント参考にして、計11台構成で作ってみたお

#まずはproxyサーバから。
#必要なモジュールの準備

apt-get install python-software-properties
add-apt-repository ppa:swift-core/release
apt-get update
apt-get install curl gcc git-core memcached python-configobj python-coverage python-dev python-nose python-setuptools python-simplejson python-xattr sqlite3 xfsprogs python-webob python-eventlet python-greenlet python-pastedeploy python-netifaces

#ユーザつくる
/usr/sbin/groupadd -g XXXXX yatta_swift
/usr/sbin/useradd -u XXXXX -g yatta_swift yatta_swift


#ディレクトリつくる
mkdir -p /etc/swift

#hashの設定ファイルつくる
cat >/etc/swift/swift.conf <> /etc/fstab
mkdir -p /srv/node/sdb1
mount -a
chown -R yatta_swift: /srv/*

#recon用のディレクトリきっとく
mkdir -p /var/cache/swift
chown -R yatta_swift: /var/cache/swift/

#proxy serverの設定 2台ともやるお

vim /etc/swift/proxy-server.conf
[DEFAULT]
#cert_file = /etc/swift/cert.crt
#key_file = /etc/swift/cert.key
bind_port = 8080
workers = 8
user = yatta_swift
log_facility = LOG_LOCAL1
[pipeline:main]
pipeline = healthcheck cache tempauth staticweb proxy-server

[app:proxy-server]
use = egg:swift#proxy
allow_account_management = true
account_autocreate = true

[filter:tempauth]
use = egg:swift#tempauth
##proxyのinternval vipを設定
user_system_root = testpass .admin https:XXX.XXX.XXX.XXX:8080/
reseller_prefix = /yatta_images/
user_test_tester = testing .admin
user_test2_tester2 = testing2 .admin
user_test_tester3 = testing3

[filter:healthcheck]
use = egg:swift#healthcheck

[filter:cache]
use = egg:swift#memcache
##proxyserverのipを追加
memcache_servers = XXX.XXX.XXX.XXX:11211,XXX.XXX.XXX.XXX:11211

#静的配信もするので、staticwebモジュール追加
[filter:staticweb]
use = egg:swift#staticweb



#ログ周りの設定
vim /etc/rsyslog.d/10-swift.conf 
# Uncomment the following to have a log containing all logs together
#local1.*   /var/log/swift/all.log

# Uncomment the following to have hourly proxy logs for stats processing
#$template HourlyProxyLog,"/var/log/swift/hourly/%$YEAR%%$MONTH%%$DAY%%$HOUR%"
#local1.*;local1.!notice ?HourlyProxyLog

local1.*;local1.!notice /var/log/swift/proxy.log
local1.notice           /var/log/swift/proxy.error
local1.*                ~

vim /etc/rsyslog.conf
$PrivDropToGroup adm

mkdir -p /var/log/swift/hourly
chown -R yatta_swift /var/log/swift
chmod -R g+w /var/log/swift
service rsyslog restart

ここから、container、account、objectの設定するお その前に、swiftの各node間の同期はrynscでやるんでproxyをのぞく各nodeで下記を設定

vim /etc/rsyncd.conf 
uid = yatta_swift
gid = yatta_swift
log file = /var/log/rsyncd.log
pid file = /var/run/rsyncd.pid
address = xxx.xxx.xx.xxx 

[account]
max connections = 100
path = /srv/node/
read only = false
lock file = /var/lock/account.lock

#rysncの設定ファイル編集
vim /etc/default/rsync 
RSYNC_ENABLE=true

#rsync再起動
/etc/init.d/rsync restart

#accountの設定の場合 ※各nodeでやる
vim  /etc/swift/account-server.conf 
[DEFAULT]
#containerのip
bind_ip = XXX.XXX.XXX.XXX
workers = 100
log_facility = LOG_LOCAL1
mount_check = false
disable_fallocate = true
recon_cache_path = /var/cache/swift
user = yatta_swift

[pipeline:main]
pipeline = recon account-server

[app:account-server]
use = egg:swift#account

[filter:recon]
use = egg:swift#recon

[account-replicator]

[account-auditor]

[account-reaper]

#containerの場合
vim /etc/swift/container-server.conf 
[DEFAULT]
bind_ip = xxx.xxx.xxx.xxx
mount_check = false
disable_fallocate = true
workers = 100
log_facility = LOG_LOCAL1
devices = /srv/node
user = yatta_swift
recon_cache_path = /var/cache/swift

[pipeline:main]
pipeline = recon container-server

[app:container-server]
use = egg:swift#container

[container-replicator]

[filter:recon]
use = egg:swift#recon

[container-updater]

[container-auditor]

[container-sync]

#objectの場合
/etc/swift/object-server.conf 
[DEFAULT]
bind_ip = xxx.xxx.xxx.xxx
workers = 100
log_facility = LOG_LOCAL1
devices = /srv/node
user = yatta_swift
mount_check = false
disable_fallocate = true
recon_cache_path = /var/cache/swift

[pipeline:main]
pipeline = recon object-server

[app:object-server]
use = egg:swift#object

[filter:recon]
use = egg:swift#recon

[object-replicator]

[object-updater]

[object-auditor]

swift-clientいれとく

python-swiftclient-1.2.0.tar.gzを本家からDL
tar zxvf python-swiftclient-1.2.0.tar.gz
python setup.py install

swift起動させてみる

#swift起動させる
#proxyの場合
swift-init proxy start

#container object accountの場合
#※ほんとは個別であげるべきだとおもうけど　とりま。
swift-init all start

#proxyでringファイル生成(複数ある場合は一台だけで)
swift-ring-builder account.builder create 18 3 1
swift-ring-builder container.builder create 18 3 1
swift-ring-builder object.builder create 18 3 1

swift-ring-builder account.builder add z1-xxx.xxx.xxx.xxx:6002/sdb1 1
swift-ring-builder account.builder add z2-xxx.xxx.xxx.xxx:6002/sdb1 1
swift-ring-builder account.builder add z3-xxx.xxx.xxx.xxx:6002/sdb1 1
swift-ring-builder account.builder rebalance
 
swift-ring-builder container.builder add z1-xxx.xxx.xxx.xxx:6001/sdb1 1
swift-ring-builder container.builder add z2-xxx.xxx.xxx.xxx:6001/sdb1 1
swift-ring-builder container.builder add z3-xxx.xxx.xxx.xxx:6001/sdb1 1
swift-ring-builder container.builder rebalance
 
swift-ring-builder object.builder add z1-xxx.xxx.xxx.xxx:6000/sdb1 1
swift-ring-builder object.builder add z2-xxx.xxx.xxx.xxx:6000/sdb1 1
swift-ring-builder object.builder add z3-xxx.xxx.xxx.xxx:6000/sdb1 1
swift-ring-builder object.builder rebalance

#builderファイルと*.ring.gzが生成されるので、builderは各proxyに*.ring.gzは全nodeの/etc/swift配下に。

ちょっといじってみるお

#各nodeの状況確認@proxyで

#swift-ring-builder container.builder 
container.builder, build version 6
262144 partitions, 3 replicas, 3 zones, 3 devices, 0.00 balance
The minimum number of hours before a partition can be reassigned is 1
Devices:    id  zone      ip address     port      name weight partitions balance meta
             0     1    xxx.xxx.xxx.xxx  6001      sdb1   1.00     262144    0.00 
             1     2    xxx.xxx.xxx.xxx  6001      sdb1   1.00     262144    0.00 
             3     3    xxx.xxx.xxx.xxx  6001      sdb1   1.00     262144    0.00 

#nodeのadd remove rebalance
#node外す
swift-ring-builder object.builder remove z1-xxx.xxx.xxx.xxx/sdb1

#node入れる(重み代えるのもこれでできる)
swift-ring-builder object.builder add z1-xxx.xxx.xxx.xxx/sdb1 1

#reblance ちなみにrebalanceは一度やると一時間はできない　↑の設定だと。
swift-ring-builder object.builder

swift-ring-builder object.builder rebalance
Reassigned 262144 (100.00%) partitions. Balance is now 726.36.
-------------------------------------------------------------------------------
NOTE: Balance of 726.36 indicates you should push this 
      ring, wait at least 1 hours, and rebalance/repush.
-------------------------------------------------------------------------------

とりあえずいったんここまでで。 ほかにも結構いろいろ検証しているので、あとでのせます。