Qualche idea

1. DevOps with Fabric
Pycon5
Simone Federici

2. $ fab taskA taskB
from fabric.api import run, env
env.hosts = ['host1', 'host2']def taskA():
run('ls')def taskB():
run('whoami')

3. $ fab mytask:roles=role1
from fabric.api import env
env.roledefs = {
'web': ['www1', 'www2', 'www3'],
'dns': ['ns1', 'ns2']}
def mytask():
run('ls /var/www')

4. $ fab
mytask:roles=role1,exclude_hosts="a;c
"from fabric.api import env, hosts, roles, run
env.roledefs = {'role1': ['b', 'c']}
@hosts('a', 'b')
@roles('role1')def mytask():
run('ls /var/www')

5. $ fab migrate update
from fabric.api import run, roles
env.roledefs = {
'db': ['db1', 'db2'],
'web': ['web1', 'web2', 'web3'],}
@roles('db')def migrate():
# Database stuff here.
pass
@roles('web')def update():
# Code updates here.
pass

6. fab deploy
from fabric.api import run, roles, executedef deploy():
execute(migrate)
execute(update)
migrate on db1
migrate on db2
update on web1
update on web2
update on web3

7. $ fab deploy:app or $ fab deploy:db
from fabric.api import run, execute, task# For example, code talking to an HTTP API, or a database, or ...from mylib import
external_datastore# This is the actual algorithm involved. It does not care about host# lists at all.def do_work():
run("something interesting on a host")# This is the user-facing task invoked on the command line.
@taskdef deploy(lookup_param):
# This is the magic you don't get with @hosts or @roles.
# Even lazy-loading roles require you to declare available roles
# beforehand. Here, the sky is the limit.
host_list = external_datastore.query(lookup_param)
# Put this dynamically generated host list together with the work to be
# done.
execute(do_work, hosts=host_list)

8. $ fab set_hosts:app do_work
from fabric.api import run, taskfrom mylib import external_datastore# Marked as a publicly visible task, but otherwise unchanged:
still just# "do the work, let somebody else worry about what hosts to run on".
@taskdef do_work():
run("something interesting on a host")
@taskdef set_hosts(lookup_param):
# Update env.hosts instead of calling execute()
env.hosts = external_datastore.query(lookup_param)

9. Combining stdout and stderr
run("cmd", pty=False, combine_stderr=True):
run("cmd", pty=False, combine_stderr=False):
run("cmd", pty=True, combine_stderr=False):

10. $ fab -H host1,host2,host3
runs_in_parallel runs_serially
from fabric.api import *
@paralleldef runs_in_parallel():
passdef runs_serially():
pass
runs_in_parallel on host1, host2, and host3
runs_serially on host1
runs_serially on host2
runs_serially on host3

11. $ fab -P -z 5 heavy_task
from fabric.api import *
@parallel(pool_size=5)def heavy_task():
# lots of heavy local lifting or lots of IO here

12. @task(alias=’short’)
from fabric.api import task, run
@taskdef mytask():
run("a command")
@task(alias='dwm')def deploy_with_migrations():
pass

13. Submodule deploy.py
@task(default=True)def full_deploy():
pass
$ fab --list
Available commands:
deploy
deploy.full_deploy
deploy.migrate
deploy.provision
deploy.push

14. Class Task
class MyTask(Task):
name = "deploy"
def run(self, environment, domain="whatever.com"):
run("git clone foo")
sudo("service apache2 restart")
instance = MyTask()
VS
@taskdef deploy(environment, domain="whatever.com"):
run("git clone foo")
sudo("service apache2 restart")

15. Colors
from fabric.colors import greenprint(green("This text is green!"))
fabric.colors.blue(text, bold=False)
fabric.colors.cyan(text, bold=False)
fabric.colors.green(text, bold=False)
fabric.colors.magenta(text, bold=False)
fabric.colors.red(text, bold=False)
fabric.colors.white(text, bold=False)
fabric.colors.yellow(text, bold=False)

16. Context managers
def mytask():
with cd('/path/to/app'), prefix('workon myvenv'):
run('./manage.py syncdb')
run('./manage.py loaddata myfixture')
with cd('/var/www'):
run('ls') # cd /var/www && ls
with cd('website1'):
run('ls') # cd /var/www/website1 && ls
with hide('running', 'stdout', 'stderr'):
run('ls /var/www')
# Map localhost:6379 on the server to localhost:6379 on the client,
# so that the remote 'redis-cli' program ends up speaking to the local
# redis-server.
with remote_tunnel(6379):
run("redis-cli -i")

17. Contrib
Django Integration
Rsync Project
Upload Project
Console Confirm y/n
Files and Directory

18. So… what is Fabric?
● Deploy
● Manage multiple server
● Clustering
● Multiplatform
● Parallel
● Testing
● SSH Authentication

19. SSH + Bash Power + Python =
Rocks**3 = Fabric

20. fab release:master
…

21. django integration
from fabric.contrib import django
django.settings_module('myproject.settings')from django.conf import settingsdef
dump_production_database():
run(pg_dump -U %s -w %s > /bck/prod-db.sql' % (
settings.DATABASE_USER,
settings.DATABASE_NAME
))

22. fab deploy
@hosts('user@example.com')def deploy():
with cd("/opt/myproject"):
run("git pull")
run("django-admin.py collectstatic --noinput")
run("django-admin.py migrate --noinput")
run("/etc/init.d/uwsgi stop || echo 'done'")
run("/etc/init.d/uwsgi start")

23. fab backup_and_publish
@task
@hosts('www-data@example.com')def backup_and_publish():
run('''tar cjvf /var/wwwbackups/www.%s.tar.bz2 --
exclude=/var/www/download --exclude=/var/www/backup* /var/www''' %
today().strftime("%Y%m%d%H%M%S"))
run('rsync -avz --checksum --ignore-times /var/wwwstage/ /var/www') #--delete

24. fab static_generation
@taskdef static_generation():
execute(remote_generation)
local("wget --user=admin --password=rootme --recursive --page-requisites --
html-extension --convert-links --restrict-file-names=windows --domains
example.com --no-parent http://wwwstage.example.com/ -o dump.log || echo 'Looking
for 404 on wwwstage.example.com'")
local("cat dump.log | grep -B 2 '404 Not Found' | grep
'http://wwwstage.example.com/' || echo 'OK no 404 found...'")

25. fab upload_release_note:2.10.0
@task
@hosts('install@example.com')def upload_release_note(version):
release_note_file_name = "RELEASE_NOTE_%s.TXT" % version
with open(release_note_file_name,"w") as out_file:
notes = jira.Query().render(version=version)
out_file.write(notes.encode('ascii', 'ignore'))
out_file.close()
put(release_note_file_name, "/cygdrive/d/%s/" % version)

26. fab cleanup_nexus:10.0.2-RC2
@taskdef cleanup_nexus(version):
for module in [
"core-api",
"core-client-rest",
"core-manual",
"core-web"]:
local("curl -X DELETE -u user:rootme
http://nexus.example.com:8180/nexus/service/local/repositories/releases/content/
example/%s/%s/" % (module, version))

27. LOCK_FILE = "~/.lockfile.release.core.lock"class Lock():
def __enter__(self):
if os.access(os.path.expanduser(LOCK_FILE), os.F_OK):
pidfile = open(os.path.expanduser(LOCK_FILE), "r")
pidfile.seek(0)
old_pid = pidfile.readline()
print "There is an already a process running with pid: %s," % old_pid
sys.exit(1)
pidfile = open(os.path.expanduser(LOCK_FILE), "w")
pidfile.write("%s" % os.getpid())
pidfile.close
def __exit__(self, type, value, traceback):
os.remove(os.path.expanduser(LOCK_FILE))
Locks

28. fab send_email_candidate:2.12,me@...
@taskdef send_mail_candidate(version, *receivers):
sender = 'development@geniusbytes.com'
body = """From: Core Team <noreply@example.com>To: Development
<development@example.com>Subject: New Release CANDIDATE %(version)snNew Release
CANDIDATE %(version)savailable on: * smb://example.com/myproject/%(version)s
""" % dict(version=version)
try:
message = smtplib.SMTP('example.com')
message.sendmail(sender, receivers, body)
print "Successfully sent email"
except smtplib.SMTPException:
print "Error: unable to send email"

29. XML parser
@taskdef get_pom_version():
src=os.path.dirname(__file__)
pom_file=os.path.abspath(os.path.join(src, 'pom.xml'))
from xml.dom.minidom import parse
pom = parse(pom_file)
version = pom.getElementsByTagName("version")[1].firstChild.nodeValue
find = re.compile(r'^d+.d+.d+-([a-zA-Z-]+)d*-SNAPSHOT$').findall(version)
if not find:
abort(version + " is not a valid development version")
versions = re.compile(r'd+').findall(version)
if len(versions) is 3:
versions.append(1)
if len(versions) is not 4:
abort(version + " is not a valid development version")
versions.extend(find)
return versions

30. fab sed_poms_version:2.13
@taskdef sed_poms_version(new_version):
major, minor, patch, rev, rc = get_pom_version()
version = "%s.%s.%s-%s%s-SNAPSHOT" % (major, minor, patch, rc, rev)
local("sed -i '' 's@%s@%s@g' pom.xml */pom.xml" % (version, new_version))

31. fab create_manuals:2.10
ACTIVATE="source /home/installer/.sphinx_env/bin/activate"
@hosts('installer@example.com')def create_manuals(version):
with cd(DEPLOY_DIR + "/myproject"):
name = "manual-%s-doc" % version
run("wget
http://nexus.example.com:8180/nexus/service/local/repo_groups/public/content/com/
myproject/manual/%s/%s.zip" % (version, name))
with cd(name):
run(ACTIVATE + "&& make docs")
run("mv target/en/latex/MyProject*.pdf docs/" % version)
run("tar cjvf docs/MyCDDS-en-%s.tar.bz2 target/docs/" % version)
run("scp -r docs install@example.com:/cygdrive/d/%s/docs" % version)

32. fab create_installer:2.10 (innosetup)
@hosts('install@example.com')def create_installer(version):
name = "installer-%s-app" % version
run("wget
http://nexus.geniusbytes.com:8180/nexus/service/local/repo_groups/public/content/
myproject/installer/%s/%s.zip" % (version, name))
with cd(name):
run("tar xjvf /cygdrive/d/%(version)s/MyCDDS-en-%(version)s.tar.bz2" %
dict(version=version))
run(ANT)
run("mkdir -p /cygdrive/d/%s" % version)
run("cp build/*.exe /cygdrive/d/%s/" % version)
run("cp build-update/*.exe /cygdrive/d/%s/" % version)
run("rm -rf %s" % name)

33. fab release_core:master,2.10,2.11
@hosts('installer@ecample.com')def release_core(branch, version, next_version):
with cd(CORE_DEPLOY_DIR):
run('git clone ssh://installer@example.com/srv/git/myproject.git')
with cd(CORE_DEPLOY_DIR + "/myproject"):
run("git checkout %s" % branch)
run("mvn clean install")
run("mvn --batch-mode release:clean release:prepare -
DreleaseVersion=%s -DdevelopmentVersion=%s" % (version, next_version))
run("mvn release:perform")

34. fab release:master
def release(branch):
with Lock():
major, minor, patch, rev, rc = check_current_version(branch)
if 'RC' != rc:
abort("RC not found, not possible release a final version")
version = "%s.%s.%s" % (major, minor, patch)
next_version = "%s.%s.%s-RC%s-SNAPSHOT" % (major, minor, int(patch)+1, 1)
puts("preparing to release %s (next will be %s)" % (version,
next_version))
execute(release_core, branch, version, next_version)
execute(create_manuals, version)
execute(create_installer, version)
execute(upload_release_note, version)
execute(send_mail_final_release, version, 'me@ex.com', 'dev@ex.com')
local("git pull")
execute(labels.missing_translations, 'me@ex.com')

35. fab cluod_remote_control
...

36. EC2 Testing with 200 micro server
EC2 + Fabric + Funkload
● EC2 use all ubuntu standard AMI
● Fabric as remote control, move files, aggregate.
● Funkload in order to stress an internet application. (not on
EC2)

37. Performance Testing Architecture
Target
Cloud
CTRL
Tester
Target
Target
FunkLoad
Fabric
(nmon+pefmon)
Fabric +
EC2
Fabric +
EC2

38. Testing phases
I. Prepare Monitoring
II.Prepare Cloud
1. Start Monitoring
2. Start Parallel Testing
3. Collecting Test Results
4. Collecting Perf Results
5. ReportingTarge
t
Cloud
CTRL
Tester
Targe
tTarge
t
FunkLoa
d
Fabric
(nmon+pefmon)
Fabric +
EC2
Fabric +
EC2

39. Testing Console
fab prepare_monitoring
fab prepare_cloud
fab start_monitoring
fab start_testing:ciccio,100,5000
fab collecting_test_results:ciccio
fab collecting_perf_results:ciccio
fab reporting:ciccio