Contenu connexe Similaire à Goptuna Distributed Bayesian Optimization Framework at Go Conference 2019 Autumn (20) Plus de Masashi Shibata (20) Goptuna Distributed Bayesian Optimization Framework at Go Conference 2019 Autumn1. Go Conference 2019 Autumn
c-bata c_bata_
Distributed Bayesian Optimization Framework
Goptuna
20. package main
import (
"log"
"math"
bayesopt "github.com/d4l3k/go-bayesopt"
)
var (
px1 = bayesopt.UniformParam{Name: "X1", Max: 10, Min: -10}
px2 = bayesopt.UniformParam{Name: "X2", Max: 10, Min: -10}
searchSpace = []bayesopt.Param{px1, px2}
)
func objective(params map[bayesopt.Param]float64) float64 {
x1 := params[px1]
x2 := params[px2]
return math.Pow(x1-1, 2) + math.Pow(x2-2, 2)
}
func main() {
o := bayesopt.New(
searchSpace,
bayesopt.WithRandomRounds(10),
bayesopt.WithRounds(100),
bayesopt.WithMinimize(true))
x, y, err := o.Optimize(objective)
if err != nil { ... }
log.Printf(...)
}
24. package main
import (
"log"
"math"
"github.com/c-bata/goptuna"
"github.com/c-bata/goptuna/tpe"
)
func objective(trial goptuna.Trial) (float64, error) {
x1, _ := trial.SuggestUniform("x1", -10, 10)
x2, _ := trial.SuggestUniform("x2", -10, 10)
return math.Pow(x1-2, 2) + math.Pow(x2+5, 2), nil
}
func main() {
study, _ := goptuna.CreateStudy(
"goptuna-example",
goptuna.StudyOptionSampler(tpe.NewSampler()))
err = study.Optimize(objective, 100)
if err != nil { ... }
v, _ := study.GetBestValue()
params, _ := study.GetBestParams()
log.Printf(...)
}
25. package main
import (
"log"
"math"
"github.com/c-bata/goptuna"
"github.com/c-bata/goptuna/tpe"
)
func objective(trial goptuna.Trial) (float64, error) {
x1, _ := trial.SuggestUniform("x1", -10, 10)
x2, _ := trial.SuggestUniform("x2", -10, 10)
return math.Pow(x1-2, 2) + math.Pow(x2+5, 2), nil
}
func main() {
study, _ := goptuna.CreateStudy(
"goptuna-example",
goptuna.StudyOptionSampler(tpe.NewSampler()))
err = study.Optimize(objective, 100)
if err != nil { ... }
v, _ := study.GetBestValue()
params, _ := study.GetBestParams()
log.Printf(...)
}
goptuna.Trial
float64 error
26. package main
import (
"log"
"math"
"github.com/c-bata/goptuna"
"github.com/c-bata/goptuna/tpe"
)
func objective(trial goptuna.Trial) (float64, error) {
x1, _ := trial.SuggestUniform("x1", -10, 10)
x2, _ := trial.SuggestUniform("x2", -10, 10)
return math.Pow(x1-2, 2) + math.Pow(x2+5, 2), nil
}
func main() {
study, _ := goptuna.CreateStudy(
"goptuna-example",
goptuna.StudyOptionSampler(tpe.NewSampler()))
err = study.Optimize(objective, 100)
if err != nil { ... }
v, _ := study.GetBestValue()
params, _ := study.GetBestParams()
log.Printf(...)
}
(x1, x2) = (2, -5)
27. package main
import (
"log"
"math"
"github.com/c-bata/goptuna"
"github.com/c-bata/goptuna/tpe"
)
func objective(trial goptuna.Trial) (float64, error) {
x1, _ := trial.SuggestUniform("x1", -10, 10)
x2, _ := trial.SuggestUniform("x2", -10, 10)
return math.Pow(x1-2, 2) + math.Pow(x2+5, 2), nil
}
func main() {
study, _ := goptuna.CreateStudy(
"goptuna-example",
goptuna.StudyOptionSampler(tpe.NewSampler()))
err = study.Optimize(objective, 100)
if err != nil { ... }
v, _ := study.GetBestValue()
params, _ := study.GetBestParams()
log.Printf(...)
}
SuggestUniform SuggestLogUniform
SuggestInt SuggestCategorical
x1 x2
[-10, 10]
28. package main
import (
"log"
"math"
"github.com/c-bata/goptuna"
"github.com/c-bata/goptuna/tpe"
)
func objective(trial goptuna.Trial) (float64, error) {
x1, _ := trial.SuggestUniform("x1", -10, 10)
x2, _ := trial.SuggestUniform("x2", -10, 10)
return math.Pow(x1-2, 2) + math.Pow(x2+5, 2), nil
}
func main() {
study, _ := goptuna.CreateStudy(
"goptuna-example",
goptuna.StudyOptionSampler(tpe.NewSampler()))
err = study.Optimize(objective, 100)
if err != nil { ... }
v, _ := study.GetBestValue()
params, _ := study.GetBestParams()
log.Printf(...)
}
29. package main
import (
"log"
"math"
"github.com/c-bata/goptuna"
"github.com/c-bata/goptuna/tpe"
)
func objective(trial goptuna.Trial) (float64, error) {
x1, _ := trial.SuggestUniform("x1", -10, 10)
x2, _ := trial.SuggestUniform("x2", -10, 10)
return math.Pow(x1-2, 2) + math.Pow(x2+5, 2), nil
}
func main() {
study, _ := goptuna.CreateStudy(
"goptuna-example",
goptuna.StudyOptionSampler(tpe.NewSampler()))
err = study.Optimize(objective, 100)
if err != nil { ... }
v, _ := study.GetBestValue()
params, _ := study.GetBestParams()
log.Printf(...)
}
30. package main
import (
"log"
"math"
"github.com/c-bata/goptuna"
"github.com/c-bata/goptuna/tpe"
)
func objective(trial goptuna.Trial) (float64, error) {
x1, _ := trial.SuggestUniform("x1", -10, 10)
x2, _ := trial.SuggestUniform("x2", -10, 10)
return math.Pow(x1-2, 2) + math.Pow(x2+5, 2), nil
}
func main() {
study, _ := goptuna.CreateStudy(
"goptuna-example",
goptuna.StudyOptionSampler(tpe.NewSampler()))
err = study.Optimize(objective, 100)
if err != nil { ... }
v, _ := study.GetBestValue()
params, _ := study.GetBestParams()
log.Printf(...)
}
31. package main
import (
"log"
"math"
"github.com/c-bata/goptuna"
"github.com/c-bata/goptuna/tpe"
)
func objective(trial goptuna.Trial) (float64, error) {
x1, _ := trial.SuggestUniform("x1", -10, 10)
x2, _ := trial.SuggestUniform("x2", -10, 10)
return math.Pow(x1-2, 2) + math.Pow(x2+5, 2), nil
}
func main() {
study, _ := goptuna.CreateStudy(
"goptuna-example",
goptuna.StudyOptionSampler(tpe.NewSampler()))
err = study.Optimize(objective, 100)
if err != nil { ... }
v, _ := study.GetBestValue()
params, _ := study.GetBestParams()
log.Printf(...)
}
$ go run _examples/simple_tpe/main.go
2019/10/28 01:55:34 [INFO] Trial finished: trialID=0 state=Complete evaluation=47.674293
2019/10/28 01:55:34 [INFO] Trial finished: trialID=1 state=Complete evaluation=16.564975
2019/10/28 01:55:34 [INFO] Trial finished: trialID=2 state=Complete evaluation=22.229764
2019/10/28 01:55:34 [INFO] Trial finished: trialID=3 state=Complete evaluation=132.160176
2019/10/28 01:55:34 [INFO] Trial finished: trialID=4 state=Complete evaluation=38.056051
:
2019/10/28 01:55:34 Best evaluation=0.038327 (x1=2.181604, x2=-4.926880)
34. $ pip install optuna bokeh mysqlclient
$ optuna dashboard
> --storage mysql+mysqldb://
goptuna:password@127.0.0.1:3306/yourdb
> -—study yourstudy
35. import ...
func main() {
db, _ := gorm.Open(“mysql",
“user:pass@tcp(localhost:3306)/yourdb" +
”?parseTime=true”)
storage := rdb.NewStorage(db)
defer db.Close()
study, _ := goptuna.LoadStudy(
“study-name",
goptuna.StudyOptionStorage(storage))
...
}
$ goptuna create-study
> --storage mysql+mysqldb://
goptuna:password@127.0.0.1:3306/yourdb
> -—study yourstudy
goputna.LoadStudy study
: _examples/concurrency
37. package main
import (
"log"
"math"
bo "github.com/d4l3k/go-bayesopt"
)
var (
px1 = bo.UniformParam{Name: "X1", Max: 10, Min: -10}
px2 = bo.UniformParam{Name: "X2", Max: 10, Min: -10}
searchSpace = []bo.Param{px1, px2}
)
func objective(params map[bo.Param]float64) float64 {
x1 := params[px1]
x2 := params[px2]
return math.Pow(x1-1, 2) + math.Pow(x2-2, 2)
}
func main() {
o := bo.New(
searchSpace,
bo.WithRandomRounds(10),
bo.WithRounds(100),
bo.WithMinimize(true))
x, y, err := o.Optimize(objective)
if err != nil { ... }
log.Printf(...)
}
package main
import (
"log"
"math"
"github.com/c-bata/goptuna"
"github.com/c-bata/goptuna/tpe"
)
func objective(trial goptuna.Trial) (float64, error) {
x1, _ := trial.SuggestUniform("x1", -10, 10)
x2, _ := trial.SuggestUniform("x2", -10, 10)
return math.Pow(x1-2, 2) + math.Pow(x2+5, 2), nil
}
func main() {
study, _ := goptuna.CreateStudy(
"goptuna-example",
goptuna.StudyOptionSampler(tpe.NewSampler()))
err = study.Optimize(objective, 100)
if err != nil { ... }
v, _ := study.GetBestValue()
params, _ := study.GetBestParams()
log.Printf(...)
}
Define-and-Run interface
go-bayesopt
Define-by-Run interface
Goptuna
38.
10 epochs
trial #1
trial #2
trial #3
trial #4
trial #5
trial #6
trial #7
trial #8
trial #9
30 epochs 90 epochs
[Jamieson and Talwalkar, 2016] [Li et al., 2018]
42. func objective(trial goptuna.Trial) (float64, error) {
// Go application
goMySQLOpenConns, _ := trial.SuggestInt("mysql_client_open_conns", 1, 64)
goMySQLIdleConns, _ := trial.SuggestInt("mysql_client_idle_conns", 1, 64)
goMySQLMaxLifetime, _ := trial.SuggestInt("mysql_client_max_lifetime", 1, 128)
goHttpIdleConnsPerHost, _ := trial.SuggestInt("http_max_idle_conns_per_host", 1, 4096)
goHttpKeepAlive, _ := trial.SuggestInt("http_keep_alive_timeout", 1, 128)
campaign, _ := trial.SuggestInt("campaign", 0, 2)
// Nginx
nginxWorkerProcesses, _ := trial.SuggestInt("nginx_worker_processes", 1, 16)
nginxWorkerConns, _ := trial.SuggestInt("nginx_worker_connections", 1, 4096)
nginxKeepAliveTimeout, _ := trial.SuggestInt("nginx_keep_alive_timeout", 1, 100)
// MySQL
innoDBBufferPoolSize, _ := trial.SuggestInt("innodb_buffer_pool_size", 10, 3800)
innoDBFlushLogAtTRXCommit, _ := trial.SuggestCategorical(“innodb_flush_log_at_trx_commit",
[]string{"0", "1", "2"})
innodbFlushMethod, _ := trial.SuggestCategorical("innodb_flush_method", []string{
"fsync",
"littlesync",
"nosync",
"O_DIRECT",
"O_DIRECT_NO_FSYNC",
})
...
} https://github.com/c-bata/goptuna-isucon9q/blob/master/optimizer/main.go
43. func objective(trial goptuna.Trial) (float64, error) {
// Go application
goMySQLOpenConns, _ := trial.SuggestInt("mysql_client_open_conns", 1, 64)
goMySQLIdleConns, _ := trial.SuggestInt("mysql_client_idle_conns", 1, 64)
goMySQLMaxLifetime, _ := trial.SuggestInt("mysql_client_max_lifetime", 1, 128)
goHttpIdleConnsPerHost, _ := trial.SuggestInt("http_max_idle_conns_per_host", 1, 4096)
goHttpKeepAlive, _ := trial.SuggestInt("http_keep_alive_timeout", 1, 128)
campaign, _ := trial.SuggestInt("campaign", 0, 2)
// Nginx
nginxWorkerProcesses, _ := trial.SuggestInt("nginx_worker_processes", 1, 16)
nginxWorkerConns, _ := trial.SuggestInt("nginx_worker_connections", 1, 4096)
nginxKeepAliveTimeout, _ := trial.SuggestInt("nginx_keep_alive_timeout", 1, 100)
// MySQL
innoDBBufferPoolSize, _ := trial.SuggestInt("innodb_buffer_pool_size", 10, 3800)
innoDBFlushLogAtTRXCommit, _ := trial.SuggestCategorical(“innodb_flush_log_at_trx_commit",
[]string{"0", "1", "2"})
innodbFlushMethod, _ := trial.SuggestCategorical("innodb_flush_method", []string{
"fsync",
"littlesync",
"nosync",
"O_DIRECT",
"O_DIRECT_NO_FSYNC",
})
...
} https://github.com/c-bata/goptuna-isucon9q/blob/master/optimizer/main.go
(database/sql).DB
44. func objective(trial goptuna.Trial) (float64, error) {
// Go application
goMySQLOpenConns, _ := trial.SuggestInt("mysql_client_open_conns", 1, 64)
goMySQLIdleConns, _ := trial.SuggestInt("mysql_client_idle_conns", 1, 64)
goMySQLMaxLifetime, _ := trial.SuggestInt("mysql_client_max_lifetime", 1, 128)
goHttpIdleConnsPerHost, _ := trial.SuggestInt("http_max_idle_conns_per_host", 1, 4096)
goHttpKeepAlive, _ := trial.SuggestInt("http_keep_alive_timeout", 1, 128)
campaign, _ := trial.SuggestInt("campaign", 0, 2)
// Nginx
nginxWorkerProcesses, _ := trial.SuggestInt("nginx_worker_processes", 1, 16)
nginxWorkerConns, _ := trial.SuggestInt("nginx_worker_connections", 1, 4096)
nginxKeepAliveTimeout, _ := trial.SuggestInt("nginx_keep_alive_timeout", 1, 100)
// MySQL
innoDBBufferPoolSize, _ := trial.SuggestInt("innodb_buffer_pool_size", 10, 3800)
innoDBFlushLogAtTRXCommit, _ := trial.SuggestCategorical(“innodb_flush_log_at_trx_commit",
[]string{"0", "1", "2"})
innodbFlushMethod, _ := trial.SuggestCategorical("innodb_flush_method", []string{
"fsync",
"littlesync",
"nosync",
"O_DIRECT",
"O_DIRECT_NO_FSYNC",
})
...
} https://github.com/c-bata/goptuna-isucon9q/blob/master/optimizer/main.go
API
(net/http).Client
45. func objective(trial goptuna.Trial) (float64, error) {
// Go application
goMySQLOpenConns, _ := trial.SuggestInt("mysql_client_open_conns", 1, 64)
goMySQLIdleConns, _ := trial.SuggestInt("mysql_client_idle_conns", 1, 64)
goMySQLMaxLifetime, _ := trial.SuggestInt("mysql_client_max_lifetime", 1, 128)
goHttpIdleConnsPerHost, _ := trial.SuggestInt("http_max_idle_conns_per_host", 1, 4096)
goHttpKeepAlive, _ := trial.SuggestInt("http_keep_alive_timeout", 1, 128)
campaign, _ := trial.SuggestInt("campaign", 0, 2)
// Nginx
nginxWorkerProcesses, _ := trial.SuggestInt("nginx_worker_processes", 1, 16)
nginxWorkerConns, _ := trial.SuggestInt("nginx_worker_connections", 1, 4096)
nginxKeepAliveTimeout, _ := trial.SuggestInt("nginx_keep_alive_timeout", 1, 100)
// MySQL
innoDBBufferPoolSize, _ := trial.SuggestInt("innodb_buffer_pool_size", 10, 3800)
innoDBFlushLogAtTRXCommit, _ := trial.SuggestCategorical(“innodb_flush_log_at_trx_commit",
[]string{"0", "1", "2"})
innodbFlushMethod, _ := trial.SuggestCategorical("innodb_flush_method", []string{
"fsync",
"littlesync",
"nosync",
"O_DIRECT",
"O_DIRECT_NO_FSYNC",
})
...
} https://github.com/c-bata/goptuna-isucon9q/blob/master/optimizer/main.go
campaign:
46. func objective(trial goptuna.Trial) (float64, error) {
// Go application
goMySQLOpenConns, _ := trial.SuggestInt("mysql_client_open_conns", 1, 64)
goMySQLIdleConns, _ := trial.SuggestInt("mysql_client_idle_conns", 1, 64)
goMySQLMaxLifetime, _ := trial.SuggestInt("mysql_client_max_lifetime", 1, 128)
goHttpIdleConnsPerHost, _ := trial.SuggestInt("http_max_idle_conns_per_host", 1, 4096)
goHttpKeepAlive, _ := trial.SuggestInt("http_keep_alive_timeout", 1, 128)
campaign, _ := trial.SuggestInt("campaign", 0, 2)
// Nginx
nginxWorkerProcesses, _ := trial.SuggestInt("nginx_worker_processes", 1, 16)
nginxWorkerConns, _ := trial.SuggestInt("nginx_worker_connections", 1, 4096)
nginxKeepAliveTimeout, _ := trial.SuggestInt("nginx_keep_alive_timeout", 1, 100)
// MySQL
innoDBBufferPoolSize, _ := trial.SuggestInt("innodb_buffer_pool_size", 10, 3800)
innoDBFlushLogAtTRXCommit, _ := trial.SuggestCategorical(“innodb_flush_log_at_trx_commit",
[]string{"0", "1", "2"})
innodbFlushMethod, _ := trial.SuggestCategorical("innodb_flush_method", []string{
"fsync",
"littlesync",
"nosync",
"O_DIRECT",
"O_DIRECT_NO_FSYNC",
})
...
} https://github.com/c-bata/goptuna-isucon9q/blob/master/optimizer/main.go
Nginx
suggest text/template
nginx.conf systemd
47. func objective(trial goptuna.Trial) (float64, error) {
// Go application
goMySQLOpenConns, _ := trial.SuggestInt("mysql_client_open_conns", 1, 64)
goMySQLIdleConns, _ := trial.SuggestInt("mysql_client_idle_conns", 1, 64)
goMySQLMaxLifetime, _ := trial.SuggestInt("mysql_client_max_lifetime", 1, 128)
goHttpIdleConnsPerHost, _ := trial.SuggestInt("http_max_idle_conns_per_host", 1, 4096)
goHttpKeepAlive, _ := trial.SuggestInt("http_keep_alive_timeout", 1, 128)
campaign, _ := trial.SuggestInt("campaign", 0, 2)
// Nginx
nginxWorkerProcesses, _ := trial.SuggestInt("nginx_worker_processes", 1, 16)
nginxWorkerConns, _ := trial.SuggestInt("nginx_worker_connections", 1, 4096)
nginxKeepAliveTimeout, _ := trial.SuggestInt("nginx_keep_alive_timeout", 1, 100)
// MySQL
innoDBBufferPoolSize, _ := trial.SuggestInt("innodb_buffer_pool_size", 10, 3800)
innoDBFlushLogAtTRXCommit, _ := trial.SuggestCategorical(“innodb_flush_log_at_trx_commit",
[]string{"0", "1", "2"})
innodbFlushMethod, _ := trial.SuggestCategorical("innodb_flush_method", []string{
"fsync",
"littlesync",
"nosync",
"O_DIRECT",
"O_DIRECT_NO_FSYNC",
})
...
} https://github.com/c-bata/goptuna-isucon9q/blob/master/optimizer/main.go
MySQL
Nginx
49. Go MySQL client (net/http).Client Nginx MySQL
SetMaxOpen
Conns
SetMaxIdleC
onns
SetConnMax
Lifetime
DialContext.
Keepalive
MaxIdleConn
sPerHost
worker_proc
esses
worker_conn
ections
keep_alive_ti
meout
innodb_buffe
r_pool_size
innodb_flush
_log_at_trx_c
ommit
innodb_flush
_method
10660
11160
(+500)
200 37 53 77s 40s 709 11 367 74s 711M 0 fsync
Go MySQL client Nginx MySQL
SetMaxOpenCo
nns
SetMaxIdleConn
s
SetConnMaxLife
time
worker_connecti
ons
innodb_buffer_p
ool_size
innodb_flush_log
_at_trx_commit
innodb_log_buff
er_size
innodb_log_file_
size
2010
2310
(+300)
50 23 4 28s 5 210M 0 27M 341M
https://github.com/c-bata/goptuna-isucon9q/issues/2
https://github.com/c-bata/goptuna-isucon9q/issues/9
54. • [Eric et al., 2010] Eric Brochu, Vlad M. Cora, and Nando de Freitas. A tutorial on Bayesian optimization of expensive cost
functions, with application to active user modeling and hierarchical reinforcement learning. 2010. arXiv:1012.2599.
• [James et al., 2011] James S. Bergstra, Remi Bardenet, Yoshua Bengio, and Balázs Kégl. Algorithms for hyper-
parameter optimization. In Advances in Neural Information Processing Systems 25. 2011.
• [Akiba at el., 2019] Takuya Akiba, Shotaro Sano, Toshihiko Yanase, Takeru Ohta, Masanori Koyama. 2019. Optuna: A
Next-generation Hyperparameter Optimization Framework. In The 25th ACM SIGKDD Conference on Knowledge
Discovery and Data Mining (KDD ’19), August 4–8, 2019.
• [Jamieson and Talwalkar, 2016] K. Jamieson and A. Talwalkar. Non-stochastic best arm identification and
hyperparameter optimization. In AISTATS, 2016.
• [Li et al., 2018] Liam Li, Kevin Jamieson, Afshin Rostamizadeh, Ekaterina Gonina, Moritz Hardt, Benjamin Recht, and
Ameet Talwalkar. Massively parallel hyperparameter tuning. arXiv preprint arXiv:1810.05934, 2018.
• [Eggensperger et al., 2013] Eggensperger, K., Feurer, M., Hutter, F., Bergstra, J., Snoek, J., Hoos, H., and Leyton-Brown,
K.: Towards an Empirical Foundation for Assessing Bayesian Optimization of Hyperparameters, in NeurIPS workshop on
Bayesian Optimization in Theory and Practice (2013).
• [Juan et al., 2016] Yu-Chin Juan, Yong Zhuang, Wei-Sheng Chin, and Chih-Jen Lin. Field-aware factorization machines
for CTR prediction. In RecSys, pages 43–50, 2016.
References