openstack swiftの構築手順。
OSはubuntu 10.04
swiftのverは1.7.0(folsum)
構成は、
LB配下にproxy:2
account:3
container:3
object:3
公式ドキュメント参考にして、計11台構成で作ってみたお
#まずはproxyサーバから。
#必要なモジュールの準備
apt-get install python-software-properties
add-apt-repository ppa:swift-core/release
apt-get update
apt-get install curl gcc git-core memcached python-configobj python-coverage python-dev python-nose python-setuptools python-simplejson python-xattr sqlite3 xfsprogs python-webob python-eventlet python-greenlet python-pastedeploy python-netifaces
#ユーザつくる
/usr/sbin/groupadd -g XXXXX yatta_swift
/usr/sbin/useradd -u XXXXX -g yatta_swift yatta_swift
#ディレクトリつくる
mkdir -p /etc/swift
#hashの設定ファイルつくる
cat >/etc/swift/swift.conf <> /etc/fstab
mkdir -p /srv/node/sdb1
mount -a
chown -R yatta_swift: /srv/*
#recon用のディレクトリきっとく
mkdir -p /var/cache/swift
chown -R yatta_swift: /var/cache/swift/
#proxy serverの設定 2台ともやるお
vim /etc/swift/proxy-server.conf
[DEFAULT]
#cert_file = /etc/swift/cert.crt
#key_file = /etc/swift/cert.key
bind_port = 8080
workers = 8
user = yatta_swift
log_facility = LOG_LOCAL1
[pipeline:main]
pipeline = healthcheck cache tempauth staticweb proxy-server
[app:proxy-server]
use = egg:swift#proxy
allow_account_management = true
account_autocreate = true
[filter:tempauth]
use = egg:swift#tempauth
##proxyのinternval vipを設定
user_system_root = testpass .admin https:XXX.XXX.XXX.XXX:8080/
reseller_prefix = /yatta_images/
user_test_tester = testing .admin
user_test2_tester2 = testing2 .admin
user_test_tester3 = testing3
[filter:healthcheck]
use = egg:swift#healthcheck
[filter:cache]
use = egg:swift#memcache
##proxyserverのipを追加
memcache_servers = XXX.XXX.XXX.XXX:11211,XXX.XXX.XXX.XXX:11211
#静的配信もするので、staticwebモジュール追加
[filter:staticweb]
use = egg:swift#staticweb
#ログ周りの設定
vim /etc/rsyslog.d/10-swift.conf
# Uncomment the following to have a log containing all logs together
#local1.* /var/log/swift/all.log
# Uncomment the following to have hourly proxy logs for stats processing
#$template HourlyProxyLog,"/var/log/swift/hourly/%$YEAR%%$MONTH%%$DAY%%$HOUR%"
#local1.*;local1.!notice ?HourlyProxyLog
local1.*;local1.!notice /var/log/swift/proxy.log
local1.notice /var/log/swift/proxy.error
local1.* ~
vim /etc/rsyslog.conf
$PrivDropToGroup adm
mkdir -p /var/log/swift/hourly
chown -R yatta_swift /var/log/swift
chmod -R g+w /var/log/swift
service rsyslog restart
ここから、container、account、objectの設定するお
その前に、swiftの各node間の同期はrynscでやるんでproxyをのぞく各nodeで下記を設定
vim /etc/rsyncd.conf
uid = yatta_swift
gid = yatta_swift
log file = /var/log/rsyncd.log
pid file = /var/run/rsyncd.pid
address = xxx.xxx.xx.xxx
[account]
max connections = 100
path = /srv/node/
read only = false
lock file = /var/lock/account.lock
#rysncの設定ファイル編集
vim /etc/default/rsync
RSYNC_ENABLE=true
#rsync再起動
/etc/init.d/rsync restart
#accountの設定の場合 ※各nodeでやる
vim /etc/swift/account-server.conf
[DEFAULT]
#containerのip
bind_ip = XXX.XXX.XXX.XXX
workers = 100
log_facility = LOG_LOCAL1
mount_check = false
disable_fallocate = true
recon_cache_path = /var/cache/swift
user = yatta_swift
[pipeline:main]
pipeline = recon account-server
[app:account-server]
use = egg:swift#account
[filter:recon]
use = egg:swift#recon
[account-replicator]
[account-auditor]
[account-reaper]
#containerの場合
vim /etc/swift/container-server.conf
[DEFAULT]
bind_ip = xxx.xxx.xxx.xxx
mount_check = false
disable_fallocate = true
workers = 100
log_facility = LOG_LOCAL1
devices = /srv/node
user = yatta_swift
recon_cache_path = /var/cache/swift
[pipeline:main]
pipeline = recon container-server
[app:container-server]
use = egg:swift#container
[container-replicator]
[filter:recon]
use = egg:swift#recon
[container-updater]
[container-auditor]
[container-sync]
#objectの場合
/etc/swift/object-server.conf
[DEFAULT]
bind_ip = xxx.xxx.xxx.xxx
workers = 100
log_facility = LOG_LOCAL1
devices = /srv/node
user = yatta_swift
mount_check = false
disable_fallocate = true
recon_cache_path = /var/cache/swift
[pipeline:main]
pipeline = recon object-server
[app:object-server]
use = egg:swift#object
[filter:recon]
use = egg:swift#recon
[object-replicator]
[object-updater]
[object-auditor]
swift-clientいれとく
python-swiftclient-1.2.0.tar.gzを本家からDL
tar zxvf python-swiftclient-1.2.0.tar.gz
python setup.py install
swift起動させてみる
#swift起動させる
#proxyの場合
swift-init proxy start
#container object accountの場合
#※ほんとは個別であげるべきだとおもうけど とりま。
swift-init all start
#proxyでringファイル生成(複数ある場合は一台だけで)
swift-ring-builder account.builder create 18 3 1
swift-ring-builder container.builder create 18 3 1
swift-ring-builder object.builder create 18 3 1
swift-ring-builder account.builder add z1-xxx.xxx.xxx.xxx:6002/sdb1 1
swift-ring-builder account.builder add z2-xxx.xxx.xxx.xxx:6002/sdb1 1
swift-ring-builder account.builder add z3-xxx.xxx.xxx.xxx:6002/sdb1 1
swift-ring-builder account.builder rebalance
swift-ring-builder container.builder add z1-xxx.xxx.xxx.xxx:6001/sdb1 1
swift-ring-builder container.builder add z2-xxx.xxx.xxx.xxx:6001/sdb1 1
swift-ring-builder container.builder add z3-xxx.xxx.xxx.xxx:6001/sdb1 1
swift-ring-builder container.builder rebalance
swift-ring-builder object.builder add z1-xxx.xxx.xxx.xxx:6000/sdb1 1
swift-ring-builder object.builder add z2-xxx.xxx.xxx.xxx:6000/sdb1 1
swift-ring-builder object.builder add z3-xxx.xxx.xxx.xxx:6000/sdb1 1
swift-ring-builder object.builder rebalance
#builderファイルと*.ring.gzが生成されるので、builderは各proxyに*.ring.gzは全nodeの/etc/swift配下に。
ちょっといじってみるお
#各nodeの状況確認@proxyで
#swift-ring-builder container.builder
container.builder, build version 6
262144 partitions, 3 replicas, 3 zones, 3 devices, 0.00 balance
The minimum number of hours before a partition can be reassigned is 1
Devices: id zone ip address port name weight partitions balance meta
0 1 xxx.xxx.xxx.xxx 6001 sdb1 1.00 262144 0.00
1 2 xxx.xxx.xxx.xxx 6001 sdb1 1.00 262144 0.00
3 3 xxx.xxx.xxx.xxx 6001 sdb1 1.00 262144 0.00
#nodeのadd remove rebalance
#node外す
swift-ring-builder object.builder remove z1-xxx.xxx.xxx.xxx/sdb1
#node入れる(重み代えるのもこれでできる)
swift-ring-builder object.builder add z1-xxx.xxx.xxx.xxx/sdb1 1
#reblance ちなみにrebalanceは一度やると一時間はできない ↑の設定だと。
swift-ring-builder object.builder
swift-ring-builder object.builder rebalance
Reassigned 262144 (100.00%) partitions. Balance is now 726.36.
-------------------------------------------------------------------------------
NOTE: Balance of 726.36 indicates you should push this
ring, wait at least 1 hours, and rebalance/repush.
-------------------------------------------------------------------------------
とりあえずいったんここまでで。
ほかにも結構いろいろ検証しているので、あとでのせます。
