SlideShare une entreprise Scribd logo
1  sur  50
Télécharger pour lire hors ligne
Twisted
Жлобич Андрей
Python Minsk Meetup – 03.2014
C10K problem
много соединений
*
поток на каждое
=
:(плохо
1 2 3
a
X Y Z
1 a X
b c
1 поток, неблокирующий IO
3 потока, блокирующий IO
block
Y
2 b Y
block
3 c Z
3 потока, блокирующий IO + GIL
1 a X
1 поток, неблокирующий IO
2 b Y 3 c Z
Event loop
started exit()False
sched[0][0]
<= time()
True
sched.pop(0)[1]()
rr, rw, _ = select(
reads.keys(), writes.keys(),
[], sched[0][0] – time())
for fd in rr: reads[fd](fd)
for fd in rw: writes[fd](fd)
Input / Output
Event loop
def mainLoop(self):
while self._started:
try:
while self._started:
self.runUntilCurrent()
t2 = self.timeout()
t = self.running and t2
self.doIteration(t)
except:
log.msg("unexpected error")
log.err()
else:
log.msg('main loop terminated')
Twisted is an event-driven
networking engine written
in Python
Twisted – это
● Асинхронное ядро (реактор)
select, poll, epoll, kqueue, cfrunloop, glib, gtk, qt, wx
● Deferred (promise + pipeline)
● Набор разнообразных абстракций
(protocol, transport, factory, endpoint, etc)
● Готовая реализация многих протоколов
(HTTP, FTP, SMTP, POP3, IMAP, SSH, DNS, IRC,
NNTP, XMPP, OSCAR, SOCKSv4, Telnet, NNTP, ...)
● Инфраструктурные решения
(демонизация, тестирование, etc)
from twisted.internet import reactor, protocol
from twisted.protocols.basic import LineReceiver
class MyChatProtocol(LineReceiver):
def connectionMade(self):
self.factory.clients.append(self)
def connectionLost(self, reason):
self.factory.clients.remove(self)
def lineReceived(self, line):
for c in self.factory.clients:
if c is not self:
c.sendLine(line)
class MyChatFactory(protocol.ServerFactory):
protocol = MyChatProtocol
def startFactory(self):
self.clients = []
reactor.listenTCP(1234, MyChatFactory())
reactor.run()
Transport – Protocol – Factory
Транспорт
прием-передача данных
tcp, udp, unix, ssl, process, loopback
Протокол
парсинг входящих данных
state уровня соединения
Фабрика
создание протоколов (reconnect etc)
state уровня приложения
from twisted.internet import reactor
from twisted.internet.protocol import Protocol, ClientCreator
class Greeter(Protocol):
def dataReceived(self, data):
print(data)
def send_message(self, msg):
self.transport.write("%srn" % msg)
def gotProtocol(g):
g.send_message("Hello!")
reactor.callLater(5, g.send_message, "How are you?")
reactor.callLater(20, g.send_message, "Bye.")
reactor.callLater(21, g.transport.loseConnection)
c = ClientCreator(reactor, Greeter)
c.connectTCP("localhost", 1234).addCallback(gotProtocol)
reactor.run()
Deferred
● Аналог Future/Promise + Pipeline
● В Deferred лежит:
– список пар (callback, errback)
– готовое значение / исключение
● Возможность приостановки
● Поддержка отмен (cancellation)
● Но нету таймаутов (раньше были)
● Печать errback в лог из __del__
import random
from twisted.internet import reactor, defer
def lowlevel_async_op(data, callback, errback):
if random.randint(0, 42):
reactor.callLater(3, callback, data)
else:
reactor.callLater(2, errback, Exception())
def do_request():
d = defer.Deferred()
lowlevel_async_op("123", d.callback, d.errback)
return d
def clean_data(data):
return int(data)
def process_data(data):
print("got %r" % data)
def log_error(f):
print("error: %s" % f)
d = do_request()
d.addCallback(clean_data)
d.addCallback(process_data)
d.addErrback(log_error)
d.addBoth(lambda _: reactor.stop())
reactor.run()
cb1 eb1
cb2 lambda x: x
lambda x: x eb3
log log
...
d.addCallbacks(cb1, eb1)
d.callback(...) d.errback(...)
d.addCallback(cb2)
d.addErrback(eb3)
d.addBoth(log)
return Deferred()
log log
sc1 se1
sc2 se2
se3sc3
def get_data():
return str(random.randint(0, 10))
def get_parsed_data():
data = get_data()
return int(data) * 2
def work_with_data():
x1 = get_parsed_data()
x2 = get_parsed_data()
print(x1 * x2)
def get_data():
d = defer.Deferred()
reactor.callLater(3,
lambda: d.callback(str(random.randint(0, 10))))
return d
def get_parsed_data():
def parse_data(data):
return int(data) * 2
return get_data().addCallback(parse_data)
def work_with_data():
def got_x1(x1):
def got_x2(x2):
print(x1 * x2)
return get_parsed_data().addCallback(got_x2)
return get_parsed_data().addCallback(got_x1)
def get_data():
return (
task.deferLater(reactor, 3, random.randint, 0, 10)
.addCallback(str))
def get_parsed_data():
return (
get_data()
.addCallback(int)
.addCallback(lambda x: x * 2))
def work_with_data():
return defer.gatherResults( # parallel!
[get_parsed_data(), get_parsed_data()]
).addCallback(
lambda xs: util.println(xs[0] * xs[1])
)
@defer.inlineCallbacks
def get_data():
yield task.deferLater(reactor, 3, int) # sleep
defer.returnValue(str(random.randint(0, 10)))
@defer.inlineCallbacks
def get_parsed_data():
data = yield get_data()
defer.returnValue(int(data) * 2)
@defer.inlineCallbacks
def work_with_data():
x1, x2 = yield defer.gatherResults(
[get_parsed_data(), get_parsed_data()])
print(x1 * x2)
ООП-интерфейсы?!
В моем любимом Python?
class IProtocol(zope.interface.Interface):
def dataReceived(data):
"""
Called whenever data is received.
"""
def connectionLost(reason):
"""
Called when the connection is shut down.
@type reason: L{twisted.python.failure.Failure}
"""
def makeConnection(transport):
"""
Make a connection to a transport and a server.
"""
def connectionMade():
"""
Called when a connection is made.
"""
twisted.internet
IAddress IConnector IResolverSimple IResolver IReactorTCP
IReactorSSL IReactorUNIX IReactorUNIXDatagram
IReactorWin32Events IReactorUDP IReactorMulticast
IReactorSocket IReactorProcess IReactorTime IDelayedCall
IReactorThreads IReactorCore IReactorPluggableResolver
IReactorDaemonize IReactorFDSet IListeningPort ILoggingContext
IFileDescriptor IReadDescriptor IWriteDescriptor
IReadWriteDescriptor IHalfCloseableDescriptor ISystemHandle
IConsumer IProducer IPushProducer IPullProducer IProtocol
IProcessProtocol IHalfCloseableProtocol IFileDescriptorReceiver
IProtocolFactory ITransport ITCPTransport IUNIXTransport
ITLSTransport ISSLTransport IProcessTransport
IServiceCollection IUDPTransport IUNIXDatagramTransport
IUNIXDatagramConnectedTransport IMulticastTransport
IStreamClientEndpoint IStreamServerEndpoint
IStreamServerEndpointStringParser
IStreamClientEndpointStringParser
twisted.cred
class ICredentials(Interface): ""
class IAnonymous(ICredentials): ""
class IUsernameDigestHash(ICredentials):
def checkHash(digestHash): ""
class IUsernamePassword(ICredentials):
def checkPassword(password): ""
class ICredentialsChecker(Interface):
credentialInterfaces = Attribute("")
def requestAvatarId(credentials): ""
class IRealm(Interface):
def requestAvatar(avatarId, mind, *interfaces): ""
class Portal:
def __init__(self, realm, checkers=()):
self.realm = realm
self.checkers = {}
for checker in checkers:
self.registerChecker(checker)
def listCredentialsInterfaces(self):
return self.checkers.keys()
def registerChecker(self, checker, *credentialInterfaces):
if not credentialInterfaces:
credentialInterfaces = checker.credentialInterfaces
for credentialInterface in credentialInterfaces:
self.checkers[credentialInterface] = checker
def login(self, credentials, mind, *interfaces):
for i in self.checkers:
if i.providedBy(credentials):
return maybeDeferred(
self.checkers[i].requestAvatarId, credentials
).addCallback(self.realm.requestAvatar, mind, *interfaces)
ifac = zope.interface.providedBy(credentials)
return defer.fail(failure.Failure(error.UnhandledCredentials(
"No checker for %s" % ', '.join(map(reflect.qual, ifac)))))
class ISiteUser(Interface):
def get_private_setting(self, pref_key): ""
def check_permission(self, perm):
def logout(self): ""
class IMailbox(Interface):
def load_incoming_mails(): ""
def send_mail(dest, message): ""
@implementer(IRealm)
class MyRealm(object):
def requestAvatar(self, avatarId, mind, *interfaces):
if ISiteUser in interfaces:
a = SiteUserImpl(avatarId, ...)
return ISiteUser, a, a.logout
if IMailbox in interfaces:
a = UserMailboxImpl(avatarId)
return ISiteUser, a, lambda: None
raise NotImplementedError
Plugin system
Разработчик приложения
● Пишем свой интерфейс
ISomeLogic
● Вызываем
twisted.plugin.getPlugins(ISomeLogic)
Разработчик плагина
● Пишем класс CustomLogic, реализующий
ISomeLogic + twisted.plugin.IPlugin
● Создаем файл twisted/plugins/xxx.py
● В файл xxx.py создаем экземпляр CustomLogic
yyy = CustomLogic()
Application framework
● Инициализация приложения
● Демонизация
● Выбор реактора
● Логирование
● Профилирование
● Упрощение конфигурации
● Не DI-фреймворк!
● Deprecated: персистентность приложения
Application framework
class IService(Interface):
def setServiceParent(parent):
""" Set the parent of the service. """
def disownServiceParent():
"""Remove L{IService} from L{IServiceCollection}."""
def privilegedStartService():
""" Do preparation work for starting the service."""
def startService():
""" Start the service. """
def stopService():
""" Stop the service. """
Application framework
● TCPServer, TCPClient
● SSLServer, SSLClient
● UDPServer, UDPClient
● UNIXServer, UNIXClient
● UNIXDatagramServer, UNIXDatagramClient
● StreamServerEndpointService
● MulticastServer
● TimerService
● CooperatorService
Application framework
Application
MultiService
MultiService
TCPClient
TCPServer
TimerServiceCustomService2
CustomService1
factory
factory
# file 'simple_pinger.py'
from twisted.internet import protocol
class SimplePinger(protocol.Protocol):
def connectionMade(self):
print("connection made")
self.factory.resetDelay()
self.factory.client = self
def connectionLost(self, reason):
print("connection lost")
self.factory.client = None
def ping(self, token):
self.transport.write("ping {0}rn".format(token))
class PingerFactory(protocol.ReconnectingClientFactory):
client = None
protocol = SimplePinger
counter = 0
def ping(self):
if self.client:
self.counter += 1
return self.client.ping(self.counter)
Application framework
# file 'pinger_1234.tac'
from twisted.application import internet, service
from simple_pinger import PingerFactory
client = PingerFactory()
timer = internet.TimerService(1, client.ping)
clcon = internet.TCPClient("localhost", 1234, client)
application = service.Application("demo")
timer.setServiceParent(application)
clcon.setServiceParent(application)
~> twistd -y pinger_1234.tac
twistd plugins
class IServiceMaker(Interface):
tapname = Attribute("A short string naming this Twisted plugin")
description = Attribute("A brief summary of the features")
options = Attribute("A C{twisted.python.usage.Options} subclass")
def makeService(options):
"""
Create and return an object providing
L{twisted.application.service.IService}.
"""
~> twistd [opts] <<tapname>> … [plugin-opts]
~> twistd -l bs.log procmon buggy-script.py
Например
Spread – perspective broker
● Удаленный вызов методов (RPC)
● Асинхронный
● Симметричный
● Своя сериализация (jelly / banana)
● Передача объектов как по ссылке
(Referencable) так и значению (Copyable)
● Передача больших объектов (Cacheable)
from twisted.spread import pb
from twisted.application import service, internet
class Formatter(pb.Referenceable):
def __init__(self, format_spec):
self.format_spec = format_spec
def remote_formatIt(self, value):
return format(value, self.format_spec)
class ServerObject(pb.Root):
def remote_getFormatter(self, format_spec):
return Formatter(format_spec)
application = service.Application("pb-server")
sobj = ServerObject()
bs = internet.TCPServer(8800, pb.PBServerFactory(sobj))
bs.setServiceParent(application)
from twisted.internet import defer, task
from twisted.spread import pb
@defer.inlineCallbacks
def main(reactor):
cf = pb.PBClientFactory()
reactor.connectTCP("localhost", 8800, cf)
root = yield cf.getRootObject()
fmt = yield root.callRemote('getFormatter', ".2f")
res = yield fmt.callRemote('formatIt', 1.2345)
print(res)
if __name__ == '__main__':
task.react(main)
Библиотеки
websocket, memcache, redis, riak,
couchdb, cassandra, postgresql,
amqp, stomp, solr, xmpp, oscar, msn,
snmp, smpp, ldap, webdav
…
любая асинхронная библиотека
с расширяемым реактором
Threads
class IReactorThreads(Interface):
def getThreadPool():
"Return the threadpool used by L{callInThread}."
def callInThread(callable, *args, **kwargs):
"Run the callable object in a separate thread."
def callFromThread(callable, *args, **kw):
"Cause a function to be executed by the reactor."
def suggestThreadPoolSize(size):
"Suggest the size of the internal threadpool."
Helpers:
● blockingCallFromThread(reactor, f, *a, **kw)
● deferToThread(f, *args, **kwargs)
● callMultipleInThread(tupleList)
adbapi
from twisted.enterprise import adbapi
from twisted.internet import defer, task
dbpool = adbapi.ConnectionPool('sqlite3',
"file.db", check_same_thread=False)
@defer.inlineCallbacks
def useless_work():
yield dbpool.runOperation(
"CREATE TABLE Users (name, nick)")
yield dbpool.runOperation(
"INSERT INTO Users (name, nick) VALUES (?, ?)",
["Andrei", "anjensan"])
nick = yield dbpool.runQuery(
"SELECT nick FROM Users WHERE name = ?",
["Andrei"])
print(nick)
adbapi
def txn_work_with_cursor(cursor, values):
# …
cursor.executemany(
"INSERT INTO Users (name, nick) VALUES (?, ?)",
values,
)
threads.blockingCallFromThread(reactor,
lambda: task.deferLater(reactor, 3, int))
# …
cursor.execute("SELECT COUNT(*) FROM Users")
return cursor.fetchone()[0]
@defer.inlineCallbacks
def useless_work2():
row = yield dbpool.runInteraction(
txn_work_with_cursor,
[("Ivan", "ivann"), ("Petr", "pettr")])
print(row)
Twisted Web
● HTTP server
– конейнер для WSGI (threads!)
– сервер и клиент XMLRPC
– статические файлы
– проксирование
– распределенный (pb)
● HTML шаблонный движок (асинхронный)
● HTTP клиент
from twisted.internet import reactor
from twisted.application import service, internet
from twisted.web.resource import Resource
from twisted.web.server import NOT_DONE_YET, Site
import time
class ClockResource(Resource):
isLeaf = True
def render_GET(self, request):
reactor.callLater(2, self._delayed_render, request)
return NOT_DONE_YET
def _delayed_render(self, request):
request.write("%s" % time.ctime())
request.finish()
resource = ClockResource()
site = Site(resource)
application = service.Application("hello-world")
internet.TCPServer(8080, site).setServiceParent(application)
class ClockResource(Resource):
def _delayed_body(self, requset):
return task.deferLater(reactor, 2, lambda: str(time.ctime()))
def send_response(self, body, request):
request.write(body)
request.finish()
def fail_response(self, fail, request):
if fail.check(defer.CancelledError):
return
request.setResponseCode(501)
request.write(str(fail))
request.finish()
def response_failed(self, err, d):
d.cancel()
err.trap(error.ConnectionDone)
def render_GET(self, request):
d = self._delayed_body(request)
request.notifyFinish().addErrback(self.response_failed, d)
d.addCallback(self.send_response, request)
d.addErrback(self.fail_response, request)
return NOT_DONE_YET
Resource tree
Site (extends HTTPFactory)
/foo
'foo'
/bar
/bar/ham
/bar/fam/.../.....
'bar'
'ham'
'fam'
/foo/<dyn>
getChild(path)
/foo/abc
'abc'
/bar/
''
isLeaf = 1
/
''
from twisted.internet import reactor
from twisted.application import internet, service
from twisted.web import static, server, twcgi, wsgi, proxy
def wsgi_hello_world(environ, start_response):
start_response('200 OK', [('Content-type','text/plain')])
return ["Hello World!"]
root = static.File("/var/www/htdocs")
root.putChild("about", static.Data("lorem ipsum", 'text/plain'))
root.putChild("doc", static.File("/usr/share/doc"))
root.putChild("cgi-bin", twcgi.CGIDirectory("/var/www/cgi-bin"))
root.putChild("tut.by", proxy.ReverseProxyResource("tut.by", 80, ""))
hw_resource = wsgi.WSGIResource(
reactor, reactor.getThreadPool(), wsgi_hello_world)
root.putChild("hello-world", hw_resource)
site = server.Site(root)
ss = internet.TCPServer(8080, site)
application = service.Application('web')
ss.setServiceParent(application)
~> twistd web --wsgi my_module.wsgi_app
~> twistd web --path ~/my/directory/
Запуск WSGI приложения
Отдача статических файлов + *.{rpy,epy}
~> twistd web --resource-script myfile.rpy
Запуск отдельного Resource (дебаг) через *.rpy
~> twistd web --class my_module.ResourceClass
Запуск отдельного Resource (дебаг) через класс
import sys
from StringIO import StringIO
from twisted.internet import reactor, task, defer
from twisted.web import client
@defer.inlineCallbacks
def send_post(reactor, url, data):
agent = client.Agent(reactor)
request_body = client.FileBodyProducer(StringIO(data))
response = yield agent.request(
'POST', url, body_producer=request_body)
response_body = yield client.readBody(response)
print("code:", response.code)
print("response_body:", response_body)
if __name__ == '__main__':
task.react(send_post, sys.argv[1:])
HTTP клиент
import sys
from twisted.internet import defer, task
from twisted.web import template
from twisted.python.filepath import FilePath
class MyElement(template.Element):
loader = template.XMLFile(FilePath("template.xml"))
@template.renderer
def title(self, request, tag):
return tag("Title")
@template.renderer
def widgets(self, request, tag):
for wn in ["first", "second", "third"]:
yield task.deferLater(
reactor, 1,
lambda: tag.clone().fillSlots(widget_name=wn))
def main(reactor):
return template.flatten(None, MyElement(), sys.stdout.write)
task.react(main)
<p xmlns:t="http://twistedmatrix.com/ns/twisted.web.template/0.1">
<p t:render="title" />
<ul>
<li t:render="widgets">
<b><t:slot name="widget_name"/></b>
</li>
</ul>
</p>
Twisted web templates
Twisted projects
● Core – Asynchronous event loop and networking framework.
● Web – An HTTP protocol implementation.
● Conch – An SSH and SFTP protocol implementation.
● Mail – An SMTP, IMAP and POP protocol implementation.
● Names – A DNS protocol implementation with client and server.
● News – An NNTP protocol implementation with client and server.
● Words – Chat and Instant Messaging.
● Runner – Process management, including an inetd server.
● Pair – Low-level networking transports and utilities.
● Trial – Unittest-compatible automated testing.
In a Nutshell, Twisted...
has had 16,823 commits made by 86 contributors
representing 196,503 lines of code
took an estimated 51 years of effort (COCOMO model)
starting with its first commit in July, 2001
http://ohloh.net/p/twisted

Contenu connexe

Tendances

Building fast interpreters in Rust
Building fast interpreters in RustBuilding fast interpreters in Rust
Building fast interpreters in RustIngvar Stepanyan
 
Rootkit on Linux X86 v2.6
Rootkit on Linux X86 v2.6Rootkit on Linux X86 v2.6
Rootkit on Linux X86 v2.6fisher.w.y
 
Oxygine 2 d objects,events,debug and resources
Oxygine 2 d objects,events,debug and resourcesOxygine 2 d objects,events,debug and resources
Oxygine 2 d objects,events,debug and resourcescorehard_by
 
Practical JavaScript Programming - Session 1/8
Practical JavaScript Programming - Session 1/8Practical JavaScript Programming - Session 1/8
Practical JavaScript Programming - Session 1/8Wilson Su
 
Djangocon11: Monkeying around at New Relic
Djangocon11: Monkeying around at New RelicDjangocon11: Monkeying around at New Relic
Djangocon11: Monkeying around at New RelicNew Relic
 
JavaScript and the AST
JavaScript and the ASTJavaScript and the AST
JavaScript and the ASTJarrod Overson
 
PyconKR 2018 Deep dive into Coroutine
PyconKR 2018 Deep dive into CoroutinePyconKR 2018 Deep dive into Coroutine
PyconKR 2018 Deep dive into CoroutineDaehee Kim
 
Practical JavaScript Programming - Session 6/8
Practical JavaScript Programming - Session 6/8Practical JavaScript Programming - Session 6/8
Practical JavaScript Programming - Session 6/8Wilson Su
 
Do we need Unsafe in Java?
Do we need Unsafe in Java?Do we need Unsafe in Java?
Do we need Unsafe in Java?Andrei Pangin
 
Down to Stack Traces, up from Heap Dumps
Down to Stack Traces, up from Heap DumpsDown to Stack Traces, up from Heap Dumps
Down to Stack Traces, up from Heap DumpsAndrei Pangin
 
Notes for xx_use_serialgc
Notes for xx_use_serialgcNotes for xx_use_serialgc
Notes for xx_use_serialgcytoshima
 
Essentials and Impactful Features of ES6
Essentials and Impactful Features of ES6Essentials and Impactful Features of ES6
Essentials and Impactful Features of ES6Riza Fahmi
 
Wwe Management System
Wwe Management SystemWwe Management System
Wwe Management SystemNeerajMudgal1
 
DEF CON 27 -OMER GULL - select code execution from using sq lite
DEF CON 27 -OMER GULL - select code execution from using sq liteDEF CON 27 -OMER GULL - select code execution from using sq lite
DEF CON 27 -OMER GULL - select code execution from using sq liteFelipe Prado
 

Tendances (17)

3
33
3
 
Why Sifu
Why SifuWhy Sifu
Why Sifu
 
Building fast interpreters in Rust
Building fast interpreters in RustBuilding fast interpreters in Rust
Building fast interpreters in Rust
 
Rootkit on Linux X86 v2.6
Rootkit on Linux X86 v2.6Rootkit on Linux X86 v2.6
Rootkit on Linux X86 v2.6
 
Oxygine 2 d objects,events,debug and resources
Oxygine 2 d objects,events,debug and resourcesOxygine 2 d objects,events,debug and resources
Oxygine 2 d objects,events,debug and resources
 
Practical JavaScript Programming - Session 1/8
Practical JavaScript Programming - Session 1/8Practical JavaScript Programming - Session 1/8
Practical JavaScript Programming - Session 1/8
 
Djangocon11: Monkeying around at New Relic
Djangocon11: Monkeying around at New RelicDjangocon11: Monkeying around at New Relic
Djangocon11: Monkeying around at New Relic
 
JavaScript and the AST
JavaScript and the ASTJavaScript and the AST
JavaScript and the AST
 
PyconKR 2018 Deep dive into Coroutine
PyconKR 2018 Deep dive into CoroutinePyconKR 2018 Deep dive into Coroutine
PyconKR 2018 Deep dive into Coroutine
 
Practical JavaScript Programming - Session 6/8
Practical JavaScript Programming - Session 6/8Practical JavaScript Programming - Session 6/8
Practical JavaScript Programming - Session 6/8
 
Do we need Unsafe in Java?
Do we need Unsafe in Java?Do we need Unsafe in Java?
Do we need Unsafe in Java?
 
Down to Stack Traces, up from Heap Dumps
Down to Stack Traces, up from Heap DumpsDown to Stack Traces, up from Heap Dumps
Down to Stack Traces, up from Heap Dumps
 
Notes for xx_use_serialgc
Notes for xx_use_serialgcNotes for xx_use_serialgc
Notes for xx_use_serialgc
 
Essentials and Impactful Features of ES6
Essentials and Impactful Features of ES6Essentials and Impactful Features of ES6
Essentials and Impactful Features of ES6
 
Wwe Management System
Wwe Management SystemWwe Management System
Wwe Management System
 
Nantes Jug - Java 7
Nantes Jug - Java 7Nantes Jug - Java 7
Nantes Jug - Java 7
 
DEF CON 27 -OMER GULL - select code execution from using sq lite
DEF CON 27 -OMER GULL - select code execution from using sq liteDEF CON 27 -OMER GULL - select code execution from using sq lite
DEF CON 27 -OMER GULL - select code execution from using sq lite
 

En vedette

Fabric для управления серверами
Fabric для управления серверамиFabric для управления серверами
Fabric для управления серверамиMaxim Kulsha
 
B.grebnicoff scan
B.grebnicoff scanB.grebnicoff scan
B.grebnicoff scaninmamaria
 
Python легко и просто. Красиво решаем повседневные задачи
Python легко и просто. Красиво решаем повседневные задачиPython легко и просто. Красиво решаем повседневные задачи
Python легко и просто. Красиво решаем повседневные задачиMaxim Kulsha
 
Беглый обзор "внутренностей" Python
Беглый обзор "внутренностей" PythonБеглый обзор "внутренностей" Python
Беглый обзор "внутренностей" PythonMaxim Kulsha
 
Presentation
PresentationPresentation
PresentationIvlita
 
Breakfast forward2013kevinmurphy
Breakfast forward2013kevinmurphyBreakfast forward2013kevinmurphy
Breakfast forward2013kevinmurphykgbmurphy
 
Medios de comunicacion
Medios de comunicacionMedios de comunicacion
Medios de comunicacioninmamaria
 
Physical is the new digital
Physical is the new digitalPhysical is the new digital
Physical is the new digitalkgbmurphy
 
Bahan organik tanah dan siklus carbon mobilisasi unsur hara
Bahan organik tanah dan siklus carbon mobilisasi unsur haraBahan organik tanah dan siklus carbon mobilisasi unsur hara
Bahan organik tanah dan siklus carbon mobilisasi unsur haraAlvin Xevier
 
Pembahasan pola tanam
Pembahasan pola tanamPembahasan pola tanam
Pembahasan pola tanamAlvin Xevier
 
Metode perkecambahan biji
Metode perkecambahan bijiMetode perkecambahan biji
Metode perkecambahan bijiAlvin Xevier
 

En vedette (12)

Fabric для управления серверами
Fabric для управления серверамиFabric для управления серверами
Fabric для управления серверами
 
B.grebnicoff scan
B.grebnicoff scanB.grebnicoff scan
B.grebnicoff scan
 
Python легко и просто. Красиво решаем повседневные задачи
Python легко и просто. Красиво решаем повседневные задачиPython легко и просто. Красиво решаем повседневные задачи
Python легко и просто. Красиво решаем повседневные задачи
 
Беглый обзор "внутренностей" Python
Беглый обзор "внутренностей" PythonБеглый обзор "внутренностей" Python
Беглый обзор "внутренностей" Python
 
Presentation
PresentationPresentation
Presentation
 
Breakfast forward2013kevinmurphy
Breakfast forward2013kevinmurphyBreakfast forward2013kevinmurphy
Breakfast forward2013kevinmurphy
 
Medios de comunicacion
Medios de comunicacionMedios de comunicacion
Medios de comunicacion
 
Physical is the new digital
Physical is the new digitalPhysical is the new digital
Physical is the new digital
 
kopi
kopikopi
kopi
 
Bahan organik tanah dan siklus carbon mobilisasi unsur hara
Bahan organik tanah dan siklus carbon mobilisasi unsur haraBahan organik tanah dan siklus carbon mobilisasi unsur hara
Bahan organik tanah dan siklus carbon mobilisasi unsur hara
 
Pembahasan pola tanam
Pembahasan pola tanamPembahasan pola tanam
Pembahasan pola tanam
 
Metode perkecambahan biji
Metode perkecambahan bijiMetode perkecambahan biji
Metode perkecambahan biji
 

Similaire à Обзор фреймворка Twisted

A Few of My Favorite (Python) Things
A Few of My Favorite (Python) ThingsA Few of My Favorite (Python) Things
A Few of My Favorite (Python) ThingsMichael Pirnat
 
Python magicmethods
Python magicmethodsPython magicmethods
Python magicmethodsdreampuf
 
DjangoCon US 2011 - Monkeying around at New Relic
DjangoCon US 2011 - Monkeying around at New RelicDjangoCon US 2011 - Monkeying around at New Relic
DjangoCon US 2011 - Monkeying around at New RelicGraham Dumpleton
 
Rootkit on linux_x86_v2.6
Rootkit on linux_x86_v2.6Rootkit on linux_x86_v2.6
Rootkit on linux_x86_v2.6scuhurricane
 
Gevent what's the point
Gevent what's the pointGevent what's the point
Gevent what's the pointseanmcq
 
Map/reduce, geospatial indexing, and other cool features (Kristina Chodorow)
Map/reduce, geospatial indexing, and other cool features (Kristina Chodorow)Map/reduce, geospatial indexing, and other cool features (Kristina Chodorow)
Map/reduce, geospatial indexing, and other cool features (Kristina Chodorow)MongoSF
 
Testing My Patience
Testing My PatienceTesting My Patience
Testing My PatienceAdam Lowry
 
ZIO: Powerful and Principled Functional Programming in Scala
ZIO: Powerful and Principled Functional Programming in ScalaZIO: Powerful and Principled Functional Programming in Scala
ZIO: Powerful and Principled Functional Programming in ScalaWiem Zine Elabidine
 
Think Async: Asynchronous Patterns in NodeJS
Think Async: Asynchronous Patterns in NodeJSThink Async: Asynchronous Patterns in NodeJS
Think Async: Asynchronous Patterns in NodeJSAdam L Barrett
 
Django - Know Your Namespace: Middleware
Django - Know Your Namespace: MiddlewareDjango - Know Your Namespace: Middleware
Django - Know Your Namespace: Middlewarehowiworkdaily
 
PyCon 2010 SQLAlchemy tutorial
PyCon 2010 SQLAlchemy tutorialPyCon 2010 SQLAlchemy tutorial
PyCon 2010 SQLAlchemy tutorialjbellis
 
掀起 Swift 的面紗
掀起 Swift 的面紗掀起 Swift 的面紗
掀起 Swift 的面紗Pofat Tseng
 
Python 내장 함수
Python 내장 함수Python 내장 함수
Python 내장 함수용 최
 
Flask patterns
Flask patternsFlask patterns
Flask patternsit-people
 
To Err Is Human
To Err Is HumanTo Err Is Human
To Err Is HumanAlex Liu
 
Refactoring to Macros with Clojure
Refactoring to Macros with ClojureRefactoring to Macros with Clojure
Refactoring to Macros with ClojureDmitry Buzdin
 
Deterministic simulation testing
Deterministic simulation testingDeterministic simulation testing
Deterministic simulation testingFoundationDB
 
Clojure Intro
Clojure IntroClojure Intro
Clojure Introthnetos
 

Similaire à Обзор фреймворка Twisted (20)

A Few of My Favorite (Python) Things
A Few of My Favorite (Python) ThingsA Few of My Favorite (Python) Things
A Few of My Favorite (Python) Things
 
Python magicmethods
Python magicmethodsPython magicmethods
Python magicmethods
 
DjangoCon US 2011 - Monkeying around at New Relic
DjangoCon US 2011 - Monkeying around at New RelicDjangoCon US 2011 - Monkeying around at New Relic
DjangoCon US 2011 - Monkeying around at New Relic
 
Rootkit on linux_x86_v2.6
Rootkit on linux_x86_v2.6Rootkit on linux_x86_v2.6
Rootkit on linux_x86_v2.6
 
Gevent what's the point
Gevent what's the pointGevent what's the point
Gevent what's the point
 
Map/reduce, geospatial indexing, and other cool features (Kristina Chodorow)
Map/reduce, geospatial indexing, and other cool features (Kristina Chodorow)Map/reduce, geospatial indexing, and other cool features (Kristina Chodorow)
Map/reduce, geospatial indexing, and other cool features (Kristina Chodorow)
 
Testing My Patience
Testing My PatienceTesting My Patience
Testing My Patience
 
ZIO: Powerful and Principled Functional Programming in Scala
ZIO: Powerful and Principled Functional Programming in ScalaZIO: Powerful and Principled Functional Programming in Scala
ZIO: Powerful and Principled Functional Programming in Scala
 
Think Async: Asynchronous Patterns in NodeJS
Think Async: Asynchronous Patterns in NodeJSThink Async: Asynchronous Patterns in NodeJS
Think Async: Asynchronous Patterns in NodeJS
 
Django - Know Your Namespace: Middleware
Django - Know Your Namespace: MiddlewareDjango - Know Your Namespace: Middleware
Django - Know Your Namespace: Middleware
 
PyCon 2010 SQLAlchemy tutorial
PyCon 2010 SQLAlchemy tutorialPyCon 2010 SQLAlchemy tutorial
PyCon 2010 SQLAlchemy tutorial
 
Java VS Python
Java VS PythonJava VS Python
Java VS Python
 
掀起 Swift 的面紗
掀起 Swift 的面紗掀起 Swift 的面紗
掀起 Swift 的面紗
 
Python 내장 함수
Python 내장 함수Python 내장 함수
Python 내장 함수
 
Flask patterns
Flask patternsFlask patterns
Flask patterns
 
To Err Is Human
To Err Is HumanTo Err Is Human
To Err Is Human
 
ssh.isdn.test
ssh.isdn.testssh.isdn.test
ssh.isdn.test
 
Refactoring to Macros with Clojure
Refactoring to Macros with ClojureRefactoring to Macros with Clojure
Refactoring to Macros with Clojure
 
Deterministic simulation testing
Deterministic simulation testingDeterministic simulation testing
Deterministic simulation testing
 
Clojure Intro
Clojure IntroClojure Intro
Clojure Intro
 

Dernier

Cloud Frontiers: A Deep Dive into Serverless Spatial Data and FME
Cloud Frontiers:  A Deep Dive into Serverless Spatial Data and FMECloud Frontiers:  A Deep Dive into Serverless Spatial Data and FME
Cloud Frontiers: A Deep Dive into Serverless Spatial Data and FMESafe Software
 
presentation ICT roal in 21st century education
presentation ICT roal in 21st century educationpresentation ICT roal in 21st century education
presentation ICT roal in 21st century educationjfdjdjcjdnsjd
 
Rising Above_ Dubai Floods and the Fortitude of Dubai International Airport.pdf
Rising Above_ Dubai Floods and the Fortitude of Dubai International Airport.pdfRising Above_ Dubai Floods and the Fortitude of Dubai International Airport.pdf
Rising Above_ Dubai Floods and the Fortitude of Dubai International Airport.pdfOrbitshub
 
Apidays New York 2024 - Accelerating FinTech Innovation by Vasa Krishnan, Fin...
Apidays New York 2024 - Accelerating FinTech Innovation by Vasa Krishnan, Fin...Apidays New York 2024 - Accelerating FinTech Innovation by Vasa Krishnan, Fin...
Apidays New York 2024 - Accelerating FinTech Innovation by Vasa Krishnan, Fin...apidays
 
Strategize a Smooth Tenant-to-tenant Migration and Copilot Takeoff
Strategize a Smooth Tenant-to-tenant Migration and Copilot TakeoffStrategize a Smooth Tenant-to-tenant Migration and Copilot Takeoff
Strategize a Smooth Tenant-to-tenant Migration and Copilot Takeoffsammart93
 
Apidays New York 2024 - The value of a flexible API Management solution for O...
Apidays New York 2024 - The value of a flexible API Management solution for O...Apidays New York 2024 - The value of a flexible API Management solution for O...
Apidays New York 2024 - The value of a flexible API Management solution for O...apidays
 
Navigating the Deluge_ Dubai Floods and the Resilience of Dubai International...
Navigating the Deluge_ Dubai Floods and the Resilience of Dubai International...Navigating the Deluge_ Dubai Floods and the Resilience of Dubai International...
Navigating the Deluge_ Dubai Floods and the Resilience of Dubai International...Orbitshub
 
Web Form Automation for Bonterra Impact Management (fka Social Solutions Apri...
Web Form Automation for Bonterra Impact Management (fka Social Solutions Apri...Web Form Automation for Bonterra Impact Management (fka Social Solutions Apri...
Web Form Automation for Bonterra Impact Management (fka Social Solutions Apri...Jeffrey Haguewood
 
[BuildWithAI] Introduction to Gemini.pdf
[BuildWithAI] Introduction to Gemini.pdf[BuildWithAI] Introduction to Gemini.pdf
[BuildWithAI] Introduction to Gemini.pdfSandro Moreira
 
Platformless Horizons for Digital Adaptability
Platformless Horizons for Digital AdaptabilityPlatformless Horizons for Digital Adaptability
Platformless Horizons for Digital AdaptabilityWSO2
 
Connector Corner: Accelerate revenue generation using UiPath API-centric busi...
Connector Corner: Accelerate revenue generation using UiPath API-centric busi...Connector Corner: Accelerate revenue generation using UiPath API-centric busi...
Connector Corner: Accelerate revenue generation using UiPath API-centric busi...DianaGray10
 
DBX First Quarter 2024 Investor Presentation
DBX First Quarter 2024 Investor PresentationDBX First Quarter 2024 Investor Presentation
DBX First Quarter 2024 Investor PresentationDropbox
 
ICT role in 21st century education and its challenges
ICT role in 21st century education and its challengesICT role in 21st century education and its challenges
ICT role in 21st century education and its challengesrafiqahmad00786416
 
Why Teams call analytics are critical to your entire business
Why Teams call analytics are critical to your entire businessWhy Teams call analytics are critical to your entire business
Why Teams call analytics are critical to your entire businesspanagenda
 
Introduction to Multilingual Retrieval Augmented Generation (RAG)
Introduction to Multilingual Retrieval Augmented Generation (RAG)Introduction to Multilingual Retrieval Augmented Generation (RAG)
Introduction to Multilingual Retrieval Augmented Generation (RAG)Zilliz
 
"I see eyes in my soup": How Delivery Hero implemented the safety system for ...
"I see eyes in my soup": How Delivery Hero implemented the safety system for ..."I see eyes in my soup": How Delivery Hero implemented the safety system for ...
"I see eyes in my soup": How Delivery Hero implemented the safety system for ...Zilliz
 
Artificial Intelligence Chap.5 : Uncertainty
Artificial Intelligence Chap.5 : UncertaintyArtificial Intelligence Chap.5 : Uncertainty
Artificial Intelligence Chap.5 : UncertaintyKhushali Kathiriya
 
TrustArc Webinar - Unlock the Power of AI-Driven Data Discovery
TrustArc Webinar - Unlock the Power of AI-Driven Data DiscoveryTrustArc Webinar - Unlock the Power of AI-Driven Data Discovery
TrustArc Webinar - Unlock the Power of AI-Driven Data DiscoveryTrustArc
 
MINDCTI Revenue Release Quarter One 2024
MINDCTI Revenue Release Quarter One 2024MINDCTI Revenue Release Quarter One 2024
MINDCTI Revenue Release Quarter One 2024MIND CTI
 

Dernier (20)

Cloud Frontiers: A Deep Dive into Serverless Spatial Data and FME
Cloud Frontiers:  A Deep Dive into Serverless Spatial Data and FMECloud Frontiers:  A Deep Dive into Serverless Spatial Data and FME
Cloud Frontiers: A Deep Dive into Serverless Spatial Data and FME
 
presentation ICT roal in 21st century education
presentation ICT roal in 21st century educationpresentation ICT roal in 21st century education
presentation ICT roal in 21st century education
 
Rising Above_ Dubai Floods and the Fortitude of Dubai International Airport.pdf
Rising Above_ Dubai Floods and the Fortitude of Dubai International Airport.pdfRising Above_ Dubai Floods and the Fortitude of Dubai International Airport.pdf
Rising Above_ Dubai Floods and the Fortitude of Dubai International Airport.pdf
 
Apidays New York 2024 - Accelerating FinTech Innovation by Vasa Krishnan, Fin...
Apidays New York 2024 - Accelerating FinTech Innovation by Vasa Krishnan, Fin...Apidays New York 2024 - Accelerating FinTech Innovation by Vasa Krishnan, Fin...
Apidays New York 2024 - Accelerating FinTech Innovation by Vasa Krishnan, Fin...
 
+971581248768>> SAFE AND ORIGINAL ABORTION PILLS FOR SALE IN DUBAI AND ABUDHA...
+971581248768>> SAFE AND ORIGINAL ABORTION PILLS FOR SALE IN DUBAI AND ABUDHA...+971581248768>> SAFE AND ORIGINAL ABORTION PILLS FOR SALE IN DUBAI AND ABUDHA...
+971581248768>> SAFE AND ORIGINAL ABORTION PILLS FOR SALE IN DUBAI AND ABUDHA...
 
Strategize a Smooth Tenant-to-tenant Migration and Copilot Takeoff
Strategize a Smooth Tenant-to-tenant Migration and Copilot TakeoffStrategize a Smooth Tenant-to-tenant Migration and Copilot Takeoff
Strategize a Smooth Tenant-to-tenant Migration and Copilot Takeoff
 
Apidays New York 2024 - The value of a flexible API Management solution for O...
Apidays New York 2024 - The value of a flexible API Management solution for O...Apidays New York 2024 - The value of a flexible API Management solution for O...
Apidays New York 2024 - The value of a flexible API Management solution for O...
 
Navigating the Deluge_ Dubai Floods and the Resilience of Dubai International...
Navigating the Deluge_ Dubai Floods and the Resilience of Dubai International...Navigating the Deluge_ Dubai Floods and the Resilience of Dubai International...
Navigating the Deluge_ Dubai Floods and the Resilience of Dubai International...
 
Web Form Automation for Bonterra Impact Management (fka Social Solutions Apri...
Web Form Automation for Bonterra Impact Management (fka Social Solutions Apri...Web Form Automation for Bonterra Impact Management (fka Social Solutions Apri...
Web Form Automation for Bonterra Impact Management (fka Social Solutions Apri...
 
[BuildWithAI] Introduction to Gemini.pdf
[BuildWithAI] Introduction to Gemini.pdf[BuildWithAI] Introduction to Gemini.pdf
[BuildWithAI] Introduction to Gemini.pdf
 
Platformless Horizons for Digital Adaptability
Platformless Horizons for Digital AdaptabilityPlatformless Horizons for Digital Adaptability
Platformless Horizons for Digital Adaptability
 
Connector Corner: Accelerate revenue generation using UiPath API-centric busi...
Connector Corner: Accelerate revenue generation using UiPath API-centric busi...Connector Corner: Accelerate revenue generation using UiPath API-centric busi...
Connector Corner: Accelerate revenue generation using UiPath API-centric busi...
 
DBX First Quarter 2024 Investor Presentation
DBX First Quarter 2024 Investor PresentationDBX First Quarter 2024 Investor Presentation
DBX First Quarter 2024 Investor Presentation
 
ICT role in 21st century education and its challenges
ICT role in 21st century education and its challengesICT role in 21st century education and its challenges
ICT role in 21st century education and its challenges
 
Why Teams call analytics are critical to your entire business
Why Teams call analytics are critical to your entire businessWhy Teams call analytics are critical to your entire business
Why Teams call analytics are critical to your entire business
 
Introduction to Multilingual Retrieval Augmented Generation (RAG)
Introduction to Multilingual Retrieval Augmented Generation (RAG)Introduction to Multilingual Retrieval Augmented Generation (RAG)
Introduction to Multilingual Retrieval Augmented Generation (RAG)
 
"I see eyes in my soup": How Delivery Hero implemented the safety system for ...
"I see eyes in my soup": How Delivery Hero implemented the safety system for ..."I see eyes in my soup": How Delivery Hero implemented the safety system for ...
"I see eyes in my soup": How Delivery Hero implemented the safety system for ...
 
Artificial Intelligence Chap.5 : Uncertainty
Artificial Intelligence Chap.5 : UncertaintyArtificial Intelligence Chap.5 : Uncertainty
Artificial Intelligence Chap.5 : Uncertainty
 
TrustArc Webinar - Unlock the Power of AI-Driven Data Discovery
TrustArc Webinar - Unlock the Power of AI-Driven Data DiscoveryTrustArc Webinar - Unlock the Power of AI-Driven Data Discovery
TrustArc Webinar - Unlock the Power of AI-Driven Data Discovery
 
MINDCTI Revenue Release Quarter One 2024
MINDCTI Revenue Release Quarter One 2024MINDCTI Revenue Release Quarter One 2024
MINDCTI Revenue Release Quarter One 2024
 

Обзор фреймворка Twisted

  • 3. 1 2 3 a X Y Z 1 a X b c 1 поток, неблокирующий IO 3 потока, блокирующий IO block Y 2 b Y block 3 c Z
  • 4. 3 потока, блокирующий IO + GIL 1 a X 1 поток, неблокирующий IO 2 b Y 3 c Z
  • 5. Event loop started exit()False sched[0][0] <= time() True sched.pop(0)[1]() rr, rw, _ = select( reads.keys(), writes.keys(), [], sched[0][0] – time()) for fd in rr: reads[fd](fd) for fd in rw: writes[fd](fd) Input / Output
  • 6. Event loop def mainLoop(self): while self._started: try: while self._started: self.runUntilCurrent() t2 = self.timeout() t = self.running and t2 self.doIteration(t) except: log.msg("unexpected error") log.err() else: log.msg('main loop terminated')
  • 7. Twisted is an event-driven networking engine written in Python
  • 8. Twisted – это ● Асинхронное ядро (реактор) select, poll, epoll, kqueue, cfrunloop, glib, gtk, qt, wx ● Deferred (promise + pipeline) ● Набор разнообразных абстракций (protocol, transport, factory, endpoint, etc) ● Готовая реализация многих протоколов (HTTP, FTP, SMTP, POP3, IMAP, SSH, DNS, IRC, NNTP, XMPP, OSCAR, SOCKSv4, Telnet, NNTP, ...) ● Инфраструктурные решения (демонизация, тестирование, etc)
  • 9. from twisted.internet import reactor, protocol from twisted.protocols.basic import LineReceiver class MyChatProtocol(LineReceiver): def connectionMade(self): self.factory.clients.append(self) def connectionLost(self, reason): self.factory.clients.remove(self) def lineReceived(self, line): for c in self.factory.clients: if c is not self: c.sendLine(line) class MyChatFactory(protocol.ServerFactory): protocol = MyChatProtocol def startFactory(self): self.clients = [] reactor.listenTCP(1234, MyChatFactory()) reactor.run()
  • 10. Transport – Protocol – Factory Транспорт прием-передача данных tcp, udp, unix, ssl, process, loopback Протокол парсинг входящих данных state уровня соединения Фабрика создание протоколов (reconnect etc) state уровня приложения
  • 11. from twisted.internet import reactor from twisted.internet.protocol import Protocol, ClientCreator class Greeter(Protocol): def dataReceived(self, data): print(data) def send_message(self, msg): self.transport.write("%srn" % msg) def gotProtocol(g): g.send_message("Hello!") reactor.callLater(5, g.send_message, "How are you?") reactor.callLater(20, g.send_message, "Bye.") reactor.callLater(21, g.transport.loseConnection) c = ClientCreator(reactor, Greeter) c.connectTCP("localhost", 1234).addCallback(gotProtocol) reactor.run()
  • 12. Deferred ● Аналог Future/Promise + Pipeline ● В Deferred лежит: – список пар (callback, errback) – готовое значение / исключение ● Возможность приостановки ● Поддержка отмен (cancellation) ● Но нету таймаутов (раньше были) ● Печать errback в лог из __del__
  • 13. import random from twisted.internet import reactor, defer def lowlevel_async_op(data, callback, errback): if random.randint(0, 42): reactor.callLater(3, callback, data) else: reactor.callLater(2, errback, Exception()) def do_request(): d = defer.Deferred() lowlevel_async_op("123", d.callback, d.errback) return d def clean_data(data): return int(data) def process_data(data): print("got %r" % data) def log_error(f): print("error: %s" % f) d = do_request() d.addCallback(clean_data) d.addCallback(process_data) d.addErrback(log_error) d.addBoth(lambda _: reactor.stop()) reactor.run()
  • 14. cb1 eb1 cb2 lambda x: x lambda x: x eb3 log log ... d.addCallbacks(cb1, eb1) d.callback(...) d.errback(...) d.addCallback(cb2) d.addErrback(eb3) d.addBoth(log)
  • 15. return Deferred() log log sc1 se1 sc2 se2 se3sc3
  • 16. def get_data(): return str(random.randint(0, 10)) def get_parsed_data(): data = get_data() return int(data) * 2 def work_with_data(): x1 = get_parsed_data() x2 = get_parsed_data() print(x1 * x2)
  • 17. def get_data(): d = defer.Deferred() reactor.callLater(3, lambda: d.callback(str(random.randint(0, 10)))) return d def get_parsed_data(): def parse_data(data): return int(data) * 2 return get_data().addCallback(parse_data) def work_with_data(): def got_x1(x1): def got_x2(x2): print(x1 * x2) return get_parsed_data().addCallback(got_x2) return get_parsed_data().addCallback(got_x1)
  • 18. def get_data(): return ( task.deferLater(reactor, 3, random.randint, 0, 10) .addCallback(str)) def get_parsed_data(): return ( get_data() .addCallback(int) .addCallback(lambda x: x * 2)) def work_with_data(): return defer.gatherResults( # parallel! [get_parsed_data(), get_parsed_data()] ).addCallback( lambda xs: util.println(xs[0] * xs[1]) )
  • 19. @defer.inlineCallbacks def get_data(): yield task.deferLater(reactor, 3, int) # sleep defer.returnValue(str(random.randint(0, 10))) @defer.inlineCallbacks def get_parsed_data(): data = yield get_data() defer.returnValue(int(data) * 2) @defer.inlineCallbacks def work_with_data(): x1, x2 = yield defer.gatherResults( [get_parsed_data(), get_parsed_data()]) print(x1 * x2)
  • 21. class IProtocol(zope.interface.Interface): def dataReceived(data): """ Called whenever data is received. """ def connectionLost(reason): """ Called when the connection is shut down. @type reason: L{twisted.python.failure.Failure} """ def makeConnection(transport): """ Make a connection to a transport and a server. """ def connectionMade(): """ Called when a connection is made. """
  • 22. twisted.internet IAddress IConnector IResolverSimple IResolver IReactorTCP IReactorSSL IReactorUNIX IReactorUNIXDatagram IReactorWin32Events IReactorUDP IReactorMulticast IReactorSocket IReactorProcess IReactorTime IDelayedCall IReactorThreads IReactorCore IReactorPluggableResolver IReactorDaemonize IReactorFDSet IListeningPort ILoggingContext IFileDescriptor IReadDescriptor IWriteDescriptor IReadWriteDescriptor IHalfCloseableDescriptor ISystemHandle IConsumer IProducer IPushProducer IPullProducer IProtocol IProcessProtocol IHalfCloseableProtocol IFileDescriptorReceiver IProtocolFactory ITransport ITCPTransport IUNIXTransport ITLSTransport ISSLTransport IProcessTransport IServiceCollection IUDPTransport IUNIXDatagramTransport IUNIXDatagramConnectedTransport IMulticastTransport IStreamClientEndpoint IStreamServerEndpoint IStreamServerEndpointStringParser IStreamClientEndpointStringParser
  • 23. twisted.cred class ICredentials(Interface): "" class IAnonymous(ICredentials): "" class IUsernameDigestHash(ICredentials): def checkHash(digestHash): "" class IUsernamePassword(ICredentials): def checkPassword(password): "" class ICredentialsChecker(Interface): credentialInterfaces = Attribute("") def requestAvatarId(credentials): "" class IRealm(Interface): def requestAvatar(avatarId, mind, *interfaces): ""
  • 24. class Portal: def __init__(self, realm, checkers=()): self.realm = realm self.checkers = {} for checker in checkers: self.registerChecker(checker) def listCredentialsInterfaces(self): return self.checkers.keys() def registerChecker(self, checker, *credentialInterfaces): if not credentialInterfaces: credentialInterfaces = checker.credentialInterfaces for credentialInterface in credentialInterfaces: self.checkers[credentialInterface] = checker def login(self, credentials, mind, *interfaces): for i in self.checkers: if i.providedBy(credentials): return maybeDeferred( self.checkers[i].requestAvatarId, credentials ).addCallback(self.realm.requestAvatar, mind, *interfaces) ifac = zope.interface.providedBy(credentials) return defer.fail(failure.Failure(error.UnhandledCredentials( "No checker for %s" % ', '.join(map(reflect.qual, ifac)))))
  • 25. class ISiteUser(Interface): def get_private_setting(self, pref_key): "" def check_permission(self, perm): def logout(self): "" class IMailbox(Interface): def load_incoming_mails(): "" def send_mail(dest, message): "" @implementer(IRealm) class MyRealm(object): def requestAvatar(self, avatarId, mind, *interfaces): if ISiteUser in interfaces: a = SiteUserImpl(avatarId, ...) return ISiteUser, a, a.logout if IMailbox in interfaces: a = UserMailboxImpl(avatarId) return ISiteUser, a, lambda: None raise NotImplementedError
  • 26. Plugin system Разработчик приложения ● Пишем свой интерфейс ISomeLogic ● Вызываем twisted.plugin.getPlugins(ISomeLogic) Разработчик плагина ● Пишем класс CustomLogic, реализующий ISomeLogic + twisted.plugin.IPlugin ● Создаем файл twisted/plugins/xxx.py ● В файл xxx.py создаем экземпляр CustomLogic yyy = CustomLogic()
  • 27. Application framework ● Инициализация приложения ● Демонизация ● Выбор реактора ● Логирование ● Профилирование ● Упрощение конфигурации ● Не DI-фреймворк! ● Deprecated: персистентность приложения
  • 28. Application framework class IService(Interface): def setServiceParent(parent): """ Set the parent of the service. """ def disownServiceParent(): """Remove L{IService} from L{IServiceCollection}.""" def privilegedStartService(): """ Do preparation work for starting the service.""" def startService(): """ Start the service. """ def stopService(): """ Stop the service. """
  • 29. Application framework ● TCPServer, TCPClient ● SSLServer, SSLClient ● UDPServer, UDPClient ● UNIXServer, UNIXClient ● UNIXDatagramServer, UNIXDatagramClient ● StreamServerEndpointService ● MulticastServer ● TimerService ● CooperatorService
  • 31. # file 'simple_pinger.py' from twisted.internet import protocol class SimplePinger(protocol.Protocol): def connectionMade(self): print("connection made") self.factory.resetDelay() self.factory.client = self def connectionLost(self, reason): print("connection lost") self.factory.client = None def ping(self, token): self.transport.write("ping {0}rn".format(token)) class PingerFactory(protocol.ReconnectingClientFactory): client = None protocol = SimplePinger counter = 0 def ping(self): if self.client: self.counter += 1 return self.client.ping(self.counter)
  • 32. Application framework # file 'pinger_1234.tac' from twisted.application import internet, service from simple_pinger import PingerFactory client = PingerFactory() timer = internet.TimerService(1, client.ping) clcon = internet.TCPClient("localhost", 1234, client) application = service.Application("demo") timer.setServiceParent(application) clcon.setServiceParent(application) ~> twistd -y pinger_1234.tac
  • 33. twistd plugins class IServiceMaker(Interface): tapname = Attribute("A short string naming this Twisted plugin") description = Attribute("A brief summary of the features") options = Attribute("A C{twisted.python.usage.Options} subclass") def makeService(options): """ Create and return an object providing L{twisted.application.service.IService}. """ ~> twistd [opts] <<tapname>> … [plugin-opts] ~> twistd -l bs.log procmon buggy-script.py Например
  • 34. Spread – perspective broker ● Удаленный вызов методов (RPC) ● Асинхронный ● Симметричный ● Своя сериализация (jelly / banana) ● Передача объектов как по ссылке (Referencable) так и значению (Copyable) ● Передача больших объектов (Cacheable)
  • 35. from twisted.spread import pb from twisted.application import service, internet class Formatter(pb.Referenceable): def __init__(self, format_spec): self.format_spec = format_spec def remote_formatIt(self, value): return format(value, self.format_spec) class ServerObject(pb.Root): def remote_getFormatter(self, format_spec): return Formatter(format_spec) application = service.Application("pb-server") sobj = ServerObject() bs = internet.TCPServer(8800, pb.PBServerFactory(sobj)) bs.setServiceParent(application)
  • 36. from twisted.internet import defer, task from twisted.spread import pb @defer.inlineCallbacks def main(reactor): cf = pb.PBClientFactory() reactor.connectTCP("localhost", 8800, cf) root = yield cf.getRootObject() fmt = yield root.callRemote('getFormatter', ".2f") res = yield fmt.callRemote('formatIt', 1.2345) print(res) if __name__ == '__main__': task.react(main)
  • 37. Библиотеки websocket, memcache, redis, riak, couchdb, cassandra, postgresql, amqp, stomp, solr, xmpp, oscar, msn, snmp, smpp, ldap, webdav … любая асинхронная библиотека с расширяемым реактором
  • 38. Threads class IReactorThreads(Interface): def getThreadPool(): "Return the threadpool used by L{callInThread}." def callInThread(callable, *args, **kwargs): "Run the callable object in a separate thread." def callFromThread(callable, *args, **kw): "Cause a function to be executed by the reactor." def suggestThreadPoolSize(size): "Suggest the size of the internal threadpool." Helpers: ● blockingCallFromThread(reactor, f, *a, **kw) ● deferToThread(f, *args, **kwargs) ● callMultipleInThread(tupleList)
  • 39. adbapi from twisted.enterprise import adbapi from twisted.internet import defer, task dbpool = adbapi.ConnectionPool('sqlite3', "file.db", check_same_thread=False) @defer.inlineCallbacks def useless_work(): yield dbpool.runOperation( "CREATE TABLE Users (name, nick)") yield dbpool.runOperation( "INSERT INTO Users (name, nick) VALUES (?, ?)", ["Andrei", "anjensan"]) nick = yield dbpool.runQuery( "SELECT nick FROM Users WHERE name = ?", ["Andrei"]) print(nick)
  • 40. adbapi def txn_work_with_cursor(cursor, values): # … cursor.executemany( "INSERT INTO Users (name, nick) VALUES (?, ?)", values, ) threads.blockingCallFromThread(reactor, lambda: task.deferLater(reactor, 3, int)) # … cursor.execute("SELECT COUNT(*) FROM Users") return cursor.fetchone()[0] @defer.inlineCallbacks def useless_work2(): row = yield dbpool.runInteraction( txn_work_with_cursor, [("Ivan", "ivann"), ("Petr", "pettr")]) print(row)
  • 41. Twisted Web ● HTTP server – конейнер для WSGI (threads!) – сервер и клиент XMLRPC – статические файлы – проксирование – распределенный (pb) ● HTML шаблонный движок (асинхронный) ● HTTP клиент
  • 42. from twisted.internet import reactor from twisted.application import service, internet from twisted.web.resource import Resource from twisted.web.server import NOT_DONE_YET, Site import time class ClockResource(Resource): isLeaf = True def render_GET(self, request): reactor.callLater(2, self._delayed_render, request) return NOT_DONE_YET def _delayed_render(self, request): request.write("%s" % time.ctime()) request.finish() resource = ClockResource() site = Site(resource) application = service.Application("hello-world") internet.TCPServer(8080, site).setServiceParent(application)
  • 43. class ClockResource(Resource): def _delayed_body(self, requset): return task.deferLater(reactor, 2, lambda: str(time.ctime())) def send_response(self, body, request): request.write(body) request.finish() def fail_response(self, fail, request): if fail.check(defer.CancelledError): return request.setResponseCode(501) request.write(str(fail)) request.finish() def response_failed(self, err, d): d.cancel() err.trap(error.ConnectionDone) def render_GET(self, request): d = self._delayed_body(request) request.notifyFinish().addErrback(self.response_failed, d) d.addCallback(self.send_response, request) d.addErrback(self.fail_response, request) return NOT_DONE_YET
  • 44. Resource tree Site (extends HTTPFactory) /foo 'foo' /bar /bar/ham /bar/fam/.../..... 'bar' 'ham' 'fam' /foo/<dyn> getChild(path) /foo/abc 'abc' /bar/ '' isLeaf = 1 / ''
  • 45. from twisted.internet import reactor from twisted.application import internet, service from twisted.web import static, server, twcgi, wsgi, proxy def wsgi_hello_world(environ, start_response): start_response('200 OK', [('Content-type','text/plain')]) return ["Hello World!"] root = static.File("/var/www/htdocs") root.putChild("about", static.Data("lorem ipsum", 'text/plain')) root.putChild("doc", static.File("/usr/share/doc")) root.putChild("cgi-bin", twcgi.CGIDirectory("/var/www/cgi-bin")) root.putChild("tut.by", proxy.ReverseProxyResource("tut.by", 80, "")) hw_resource = wsgi.WSGIResource( reactor, reactor.getThreadPool(), wsgi_hello_world) root.putChild("hello-world", hw_resource) site = server.Site(root) ss = internet.TCPServer(8080, site) application = service.Application('web') ss.setServiceParent(application)
  • 46. ~> twistd web --wsgi my_module.wsgi_app ~> twistd web --path ~/my/directory/ Запуск WSGI приложения Отдача статических файлов + *.{rpy,epy} ~> twistd web --resource-script myfile.rpy Запуск отдельного Resource (дебаг) через *.rpy ~> twistd web --class my_module.ResourceClass Запуск отдельного Resource (дебаг) через класс
  • 47. import sys from StringIO import StringIO from twisted.internet import reactor, task, defer from twisted.web import client @defer.inlineCallbacks def send_post(reactor, url, data): agent = client.Agent(reactor) request_body = client.FileBodyProducer(StringIO(data)) response = yield agent.request( 'POST', url, body_producer=request_body) response_body = yield client.readBody(response) print("code:", response.code) print("response_body:", response_body) if __name__ == '__main__': task.react(send_post, sys.argv[1:]) HTTP клиент
  • 48. import sys from twisted.internet import defer, task from twisted.web import template from twisted.python.filepath import FilePath class MyElement(template.Element): loader = template.XMLFile(FilePath("template.xml")) @template.renderer def title(self, request, tag): return tag("Title") @template.renderer def widgets(self, request, tag): for wn in ["first", "second", "third"]: yield task.deferLater( reactor, 1, lambda: tag.clone().fillSlots(widget_name=wn)) def main(reactor): return template.flatten(None, MyElement(), sys.stdout.write) task.react(main) <p xmlns:t="http://twistedmatrix.com/ns/twisted.web.template/0.1"> <p t:render="title" /> <ul> <li t:render="widgets"> <b><t:slot name="widget_name"/></b> </li> </ul> </p> Twisted web templates
  • 49. Twisted projects ● Core – Asynchronous event loop and networking framework. ● Web – An HTTP protocol implementation. ● Conch – An SSH and SFTP protocol implementation. ● Mail – An SMTP, IMAP and POP protocol implementation. ● Names – A DNS protocol implementation with client and server. ● News – An NNTP protocol implementation with client and server. ● Words – Chat and Instant Messaging. ● Runner – Process management, including an inetd server. ● Pair – Low-level networking transports and utilities. ● Trial – Unittest-compatible automated testing.
  • 50. In a Nutshell, Twisted... has had 16,823 commits made by 86 contributors representing 196,503 lines of code took an estimated 51 years of effort (COCOMO model) starting with its first commit in July, 2001 http://ohloh.net/p/twisted