Chi trova Fabric non lo lascia più

1. Fabric al PyCon 5 di Firenze
Taa taa chi trova fabric non lo lascia più…

2. $ fab taskA taskB
from fabric.api import run, env
env.hosts = ['host1', ‘host2']
def taskA():
run(‘ls')
def taskB():
run('whoami')

3. $ fab mytask:roles=role1
from fabric.api import env
env.roledefs = {
'web': ['www1', 'www2', 'www3'],
'dns': ['ns1', ‘ns2']
}
def mytask():
run('ls /var/www')

4. $ fab mytask:roles=role1,exclude_hosts="a;c"
from fabric.api import env, hosts, roles, run
env.roledefs = {'role1': ['b', ‘c']}
@hosts('a', ‘b')
@roles(‘role1')
def mytask():
run('ls /var/www')

5. $ fab migrate update
from fabric.api import run, roles
env.roledefs = {
'db': ['db1', 'db2'],
'web': ['web1', 'web2', ‘web3'],
}
@roles(‘db')
def migrate():
# Database stuff here.
pass
@roles(‘web')
def update():
# Code updates here.
pass

6. fab deploy
from fabric.api import run, roles, execute
def deploy():
execute(migrate)
execute(update)
migrate on db1
migrate on db2
update on web1
update on web2
update on web3

7. $ fab deploy:app or $ fab deploy:db
from fabric.api import run, execute, task
from mylib import external_datastore
def do_work():
run("something interesting on a host”)
@task
def deploy(lookup_param):
host_list = external_datastore.query(lookup_param)
execute(do_work, hosts=host_list)

8. $ fab set_hosts:app do_work
from fabric.api import run, task
from mylib import external_datastore
@task
def do_work():
run("something interesting on a host")
@task
def set_hosts(lookup_param):
env.hosts = external_datastore.query(lookup_param)

9. Combining stdout and stderr
run("cmd", pty=False, combine_stderr=True):
run("cmd", pty=False, combine_stderr=False):
run("cmd", pty=True, combine_stderr=False):

10. $ fab -H host1,host2,host3 runs_in_parallel runs_serially
from fabric.api import *
@parallel
def runs_in_parallel():
pass
def runs_serially():
pass
runs_in_parallel on host1, host2, and host3
runs_serially on host1
runs_serially on host2
runs_serially on host3

11. $ fab -P -z 5 heavy_task
from fabric.api import *
@parallel(pool_size=5)
def heavy_task():
# lots of heavy local lifting or lots of IO here

12. @task(alias=’short’)
from fabric.api import task, run
@task
def mytask():
run("a command")
@task(alias=‘dwm’)
def deploy_with_migrations():
pass

13. Submodule deploy.py
@task(default=True)
def full_deploy():
pass
$ fab --list
Available commands:
deploy
deploy.full_deploy
deploy.migrate

14. Class Task
class MyTask(Task):
name = "deploy"
def run(self, environment, domain="whatever.com"):
run("git clone foo")
sudo("service apache2 restart")
instance = MyTask()
VS
@task
def deploy(environment, domain="whatever.com"):
run("git clone foo")
sudo("service apache2 restart")

15. Colors
from fabric.colors import green
print(green("This text is green!"))
fabric.colors.blue(text, bold=False)
fabric.colors.cyan(text, bold=False)
fabric.colors.green(text, bold=False)
fabric.colors.magenta(text, bold=False)
fabric.colors.red(text, bold=False)
fabric.colors.white(text, bold=False)
fabric.colors.yellow(text, bold=False)

16. Context managers
def mytask():
with cd('/path/to/app'), prefix('workon myvenv'):
run('./manage.py syncdb')
run('./manage.py loaddata myfixture')
with cd('/var/www'):
run('ls') # cd /var/www && ls
with cd('website1'):
run('ls') # cd /var/www/website1 && ls
with hide('running', 'stdout', 'stderr'):
run('ls /var/www')
# Map localhost:6379 on the server to localhost:6379 on the client,
# so that the remote 'redis-cli' program ends up speaking to the local
# redis-server.
with remote_tunnel(6379):
run("redis-cli -i")

17. Contrib
Django Integration
Rsync Project
Upload Project
Console Confirm y/n
Files and Directory

18. So… what is Fabric?
● Deploy
● Manage multiple server
● Clustering
● Multiplatform
● Parallel
● Testing
● SSH Authentication

19. SSH + Bash Power + Python = Rocks**3 = Fabric

20. fab release:master
…

21. django integration
from fabric.contrib import django
django.settings_module(‘myproject.settings')
from django.conf import settings
def dump_production_database():
run(pg_dump -U %s -w %s > /bck/prod-db.sql' % (
settings.DATABASE_USER,
settings.DATABASE_NAME
))

22. fab deploy
@hosts('user@example.com')
def deploy():
with cd("/opt/myproject"):
run("git pull")
run("django-admin.py collectstatic --noinput")
run("django-admin.py migrate --noinput")
run("/etc/init.d/uwsgi stop || echo 'done'")
run("/etc/init.d/uwsgi start")

23. fab backup_and_publish
@task
@hosts('www-data@example.com')
def backup_and_publish():
run('''tar cjvf /var/wwwbackups/www.%s.tar.bz2
--exclude=/var/www/download
—exclude=/var/www/backup* /var/www'''
% today().strftime("%Y%m%d%H%M%S"))
run('rsync -avz --checksum --ignore-times
/var/wwwstage/ /var/www') #--delete

24. fab static_generation
@task
def static_generation():
execute(remote_generation)
local("wget --user=admin --password=rootme --recursive --page-requisites --html-
extension --convert-links --restrict-file-names=windows --domains example.com --
no-parent http://wwwstage.example.com/ -o dump.log || echo 'Looking for 404 on
wwwstage.example.com'")
local("cat dump.log | grep -B 2 '404 Not Found' | grep 'http://
wwwstage.example.com/' || echo 'OK no 404 found...'")

25. fab upload_release_note:2.10.0
@task
@hosts('install@example.com')
def upload_release_note(version):
release_note_file_name = "RELEASE_NOTE_%s.TXT" % version
with open(release_note_file_name,"w") as out_file:
notes = jira.Query().render(version=version)
out_file.write(notes.encode('ascii', 'ignore'))
out_file.close()
put(release_note_file_name, "/cygdrive/d/%s/" % version)

26. fab cleanup_nexus:10.0.2-RC2
@task
def cleanup_nexus(version):
for module in [
"core-api",
"core-client-rest",
"core-manual",
"core-web"]:
local("curl -X DELETE -u user:rootme http://nexus.example.com:
8180/nexus/service/local/repositories/releases/content/example/%s/
%s/" % (module, version))

27. LOCK_FILE = “~/.lockfile.release.core.lock"
class Lock():
def __enter__(self):
if os.access(os.path.expanduser(LOCK_FILE), os.F_OK):
pidfile = open(os.path.expanduser(LOCK_FILE), "r")
pidfile.seek(0)
old_pid = pidfile.readline()
print "There is an already a process running with pid: %s," % old_pid
sys.exit(1)
pidfile = open(os.path.expanduser(LOCK_FILE), "w")
pidfile.write("%s" % os.getpid())
pidfile.close
def __exit__(self, type, value, traceback):
os.remove(os.path.expanduser(LOCK_FILE))
Locks

28. fab send_email_candidate:2.12,me@...
@task
def send_mail_candidate(version, *receivers):
sender = 'development@geniusbytes.com'
body = """From: Core Team <noreply@example.com>To: Development
<development@example.com>Subject: New Release CANDIDATE %(version)snNew
Release CANDIDATE %(version)savailable on: * smb://example.com/myproject/%
(version)s """ % dict(version=version)
try:
message = smtplib.SMTP('example.com')
message.sendmail(sender, receivers, body)
print "Successfully sent email"
except smtplib.SMTPException:
print "Error: unable to send email"

29. XML parser
@task
def get_pom_version():
src=os.path.dirname(__file__)
pom_file=os.path.abspath(os.path.join(src, 'pom.xml'))
from xml.dom.minidom import parse
pom = parse(pom_file)
version = pom.getElementsByTagName("version")[1].firstChild.nodeValue
find = re.compile(r'^d+.d+.d+-([a-zA-Z-]+)d*-SNAPSHOT$').findall(version)
if not find:
abort(version + " is not a valid development version")
versions = re.compile(r'd+').findall(version)
if len(versions) is 3:
versions.append(1)
if len(versions) is not 4:
abort(version + " is not a valid development version")
versions.extend(find)
return versions

30. fab sed_poms_version:2.13
@task
def sed_poms_version(new_version):
major, minor, patch, rev, rc = get_pom_version()
version = "%s.%s.%s-%s%s-SNAPSHOT" %
(major, minor, patch, rc, rev)
local("sed -i '' 's@%s@%s@g' pom.xml */pom.xml" %
(version, new_version))

31. fab create_manuals:2.10
ACTIVATE="source /home/installer/.sphinx_env/bin/activate"
@hosts('installer@example.com')def create_manuals(version):
with cd(DEPLOY_DIR + "/myproject"):
name = "manual-%s-doc" % version
run("wget http://nexus.example.com:8180/nexus/service/local/repo_groups/public/content/com/
myproject/manual/%s/%s.zip" % (version, name))
with cd(name):
run(ACTIVATE + "&& make docs")
run("mv target/en/latex/MyProject*.pdf docs/" % version)
run("tar cjvf docs/MyCDDS-en-%s.tar.bz2 target/docs/" % version)
run("scp -r docs install@example.com:/cygdrive/d/%s/docs" % version)

32. fab create_installer:2.10 (innosetup)
@hosts('install@example.com')def create_installer(version):
name = "installer-%s-app" % version
run("wget http://nexus.geniusbytes.com:8180/nexus/service/local/repo_groups/public/content/
myproject/installer/%s/%s.zip" % (version, name))
with cd(name):
run("tar xjvf /cygdrive/d/%(version)s/MyCDDS-en-%(version)s.tar.bz2" % dict(version=version))
run(ANT)
run("mkdir -p /cygdrive/d/%s" % version)
run("cp build/*.exe /cygdrive/d/%s/" % version)
run("cp build-update/*.exe /cygdrive/d/%s/" % version)
run("rm -rf %s" % name)

33. fab release_core:master,2.10,2.11
@hosts('installer@ecample.com')
def release_core(branch, version, next_version):
with cd(CORE_DEPLOY_DIR):
run('git clone ssh://i@example.com/srv/git/myproject.git')
with cd(CORE_DEPLOY_DIR + "/myproject"):
run("git checkout %s" % branch)
run("mvn clean install")
run("mvn --batch-mode release:clean release:prepare
-DreleaseVersion=%s -DdevelopmentVersion=%s" %
(version, next_version))
run("mvn release:perform")

34. fab release:master
def release(branch):
with Lock():
major, minor, patch, rev, rc = check_current_version(branch)
if 'RC' != rc:
abort("RC not found, not possible release a final version")
version = "%s.%s.%s" % (major, minor, patch)
next_version = "%s.%s.%s-RC%s-SNAPSHOT" % (major, minor, int(patch)+1, 1)
puts("preparing to release %s (next will be %s)" % (version, next_version))
execute(release_core, branch, version, next_version)
execute(create_manuals, version)
execute(create_installer, version)
execute(upload_release_note, version)
execute(send_mail_final_release, version, 'me@ex.com', 'dev@ex.com')
local("git pull")
execute(labels.missing_translations, 'me@ex.com')

35. fab cluod_remote_control
...

36. EC2 Testing with 200 micro server
EC2 + Fabric + Funkload
● EC2 use all ubuntu standard AMI
● Fabric as remote control, move files, aggregate.
● Funkload in order to stress an internet application. (not on
EC2)

37. Performance Testing Architecture
Target
Cloud
CTRL
Tester
Target
Target
FunkLoad
Fabric
(nmon+pefmon)
Fabric +
EC2
Fabric +
EC2

38. Testing phases
I. Prepare Monitoring
II.Prepare Cloud
1. Start Monitoring
2. Start Parallel Testing
3. Collecting Test Results
4. Collecting Perf Results
5. Reporting
Target
Cloud
CTRL
Tester
Target
Target
FunkLoa
d
Fabric
(nmon+pefmon)
Fabric +
EC2
Fabric +
EC2

39. Testing Console
fab prepare_monitoring
fab prepare_cloud
fab start_monitoring
fab start_testing:ciccio,100,5000
fab collecting_test_results:ciccio
fab collecting_perf_results:ciccio
fab reporting:ciccio