This is slides for Gunosy.go#4. http://gunosygo.connpass.com/event/7290/

1. go#package
@takufukushima

2. go!package!hierarchy
• ast
• build
• doc
• format
• parser
• printer
• scanner
• token

3. go!package:!Sta+c!analysis
• ast
• build
• doc
• format
• parser
• printer
• scanner
• token

4. go!package:!U*li*es
• ast
• build
• doc
• format
• parser
• printer
• scanner
• token

5. Sta$c&analysis

7. Typical(compiler
1. Lexical(analysis((scanning;(tokenizing)
2. Parsing
• Concrete,syntax,tree,(CST,,parse,tree)
3. Seman8c(analysis
• Abstract,syntax,tree,(AST)
4. Code(genera8on
5. Code,op=miza=on

8. go!package
1. Lexical(analysis((scanning;(tokenizing)
2. Parsing
• Concrete,syntax,tree,(CST,,parse,tree)
3. Seman8c(analysis,(Included,in,the,parser)
• Abstract,syntax,tree,(AST)
4. ___(genera8on

9. ___"genera(on
• Documenta+on,from,the,comment
• String,representa+on,of,AST
• Forma8ed,code
• Package,informa+on,for,the,build

10. Sta$c&analysis
1. token
2. scanner
3. ast
4. parser

11. token
1. token
2. scanner
3. ast
4. parser

12. Token
// Token is the set of lexical tokens of the Go programming language.
type Token int
// The list of tokens.
const (
// Special tokens
ILLEGAL Token = iota
EOF
COMMENT
literal_beg
// Identifiers and basic type literals
// (these tokens stand for classes of literals)
IDENT // main
INT // 12345
FLOAT // 123.45
IMAG // 123.45i
CHAR // 'a'
STRING // "abc"
literal_end
...

13. Tokens
var tokens = [...]string{
ILLEGAL: "ILLEGAL",
EOF: "EOF",
COMMENT: "COMMENT",
IDENT: "IDENT",
INT: "INT",
FLOAT: "FLOAT",
IMAG: "IMAG",
CHAR: "CHAR",
STRING: "STRING",
ADD: "+",
SUB: "-",
MUL: "*",
QUO: "/",
REM: "%",
...

14. Tokens'in'BNF
AST
Expression = UnaryExpr | Expression binary_op UnaryExpr .
UnaryExpr = PrimaryExpr | unary_op UnaryExpr .
Tokens
binary_op = "||" | "&&" | rel_op | add_op | mul_op .
rel_op = "==" | "!=" | "<" | "<=" | ">" | ">=" .
add_op = "+" | "-" | "|" | "^" .
mul_op = "*" | "/" | "%" | "<<" | ">>" | "&" | "&^" .
unary_op = "+" | "-" | "!" | "^" | "*" | "&" | "<-" .

15. token!package!provides!basic!data!
structures!for!scanner
• Position
• Pos
• File
• FileSet

16. Position!and!Pos
// Position describes an arbitrary source position
// including the file, line, and column location.
// A Position is valid if the line number is > 0.
type Position struct {
// filename, if any
Filename string
// offset, starting at 0
Offset int
// line number, starting at 1
Line int
// column number, starting at 1 (character count)
Column int
}
// Offset in FileSet
type Pos int

17. File
// A File is a handle for a file belonging to a FileSet.
// A File has a name, size, and line offset table.
type File struct {
set *FileSet
// file name as provided to AddFile
name string
// Pos value range for this file is
// [base...base+size]
base int
// file size as provided to AddFile
size int
// lines and infos are protected by set.mutex
// lines contains the offset of the first character
// for each line (the first entry is always 0)
lines []int
infos []lineInfo
}

18. FileSet
// A FileSet represents a set of source files.
// Methods of file sets are synchronized; multiple
// goroutines may invoke them concurrently.
type FileSet struct {
// protects the file set
mutex sync.RWMutex
// base offset for the next file
base int
// list of files in the order added to the set
files []*File
// cache of last file looked up
last *File
}

19. FileSet!methods
// AddFile adds a new file with a given filename, base offset,
// and file size to the file set s and returns the file.
func (s *FileSet) AddFile(filename string, base, size int) *File
// Read calls decode to deserialize a file set into s; s must
// not be nil.
func (s *FileSet) Read(decode func(interface{}) error) error
// Write calls encode to serialize the file set s.
func (s *FileSet) Write(encode func(interface{}) error) error

20. scanner
1. token
2. scanner
3. ast
4. parser

21. Scanner
type Scanner struct {
// immutable state
file *token.File // source file handle
dir string // directory portion of file.Name()
src []byte // source
err ErrorHandler // error reporting; or nil
mode Mode // scanning mode
// scanning state
ch rune // current character
offset int // character offset
rdOffset int // reading offset (position after current character)
lineOffset int // current line offset
insertSemi bool // insert a semicolon before next newline
// public state - ok to modify
ErrorCount int // number of errors encountered
}

22. Scanner!methods
func (s *Scanner) Init(file *token.File,
src []byte,
err ErrorHandler,
mode Mode)
// Scan scans the next token and returns the token
// position, the token, and its literal string if
// applicable. The source end is indicated by
// token.EOF.
func (s *Scanner) Scan() (pos token.Pos,
tok token.Token,
lit string)

23. Mode
type Mode uint
const (
// return comments as COMMENT tokens
ScanComments Mode = 1 << iota
// do not automatically insert semicolons
// - for testing only
dontInsertSemis
)

24. ErrorHandler,"Error"and"
ErrorList
// An ErrorHandler may be provided to Scanner.Init. If a
// syntax error is encountered and a handler was installed,
// the handler is called with a position and an error
// message. The position points to the beginning of the
// offending token.
type ErrorHandler func(pos token.Position, msg string)
type Error struct {
Pos token.Position
Msg string
}
type ErrorList []*Error

25. Usage&example&in&
scanner_test.go
func ExampleScanner_Scan() {
// src is the input that we want to tokenize.
src := []byte("cos(x) + 1i*sin(x) // Euler")
// Initialize the scanner.
var s scanner.Scanner
fset := token.NewFileSet() // positions are relative to fset
file := fset.AddFile("", fset.Base(), len(src)) // register input "file"
s.Init(file, src, nil /* no error handler */, scanner.ScanComments)
// Repeated calls to Scan yield the token sequence found in the input.
for {
pos, tok, lit := s.Scan()
if tok == token.EOF { break }
fmt.Printf("%st%st%qn", fset.Position(pos), tok, lit)
}
}

26. Result'of'the'usage'example
// output:
// 1:1 IDENT "cos"
// 1:4 ( ""
// 1:5 IDENT "x"
// 1:6 ) ""
// 1:8 + ""
// 1:10 IMAG "1i"
// 1:12 * ""
// 1:13 IDENT "sin"
// 1:16 ( ""
// 1:17 IDENT "x"
// 1:18 ) ""
// 1:20 ; "n"
// 1:20 COMMENT "// Euler"

27. ast
1. token
2. scanner
3. ast
4. parser

28. Syntax
Read%the%spec:"h$p://golang.org/ref/spec

30. Interfaces
// All node types implement the Node interface.
type Node interface {
Pos() token.Pos // position of first character belonging to the node
End() token.Pos // position of first character immediately after the node
}
// All expression nodes implement the Expr interface.
type Expr interface {
Node
exprNode()
}
// All statement nodes implement the Stmt interface.
type Stmt interface {
Node
stmtNode()
}
// All declaration nodes implement the Decl interface.
type Decl interface {
Node
declNode()
}

31. Interfaces
• Node
• Expression
• Statement
• Declara4on
• Comment6and6CommentGroup
• File6and6Package

32. U"lity'func"ons
func Walk(v Visitor, node Node)
func Fprint(w io.Writer, fset *token.FileSet, x interface{},
f FieldFilter) (err error)
func Print(fset *token.FileSet, x interface{}) error
func FilterDecl(decl Decl, f Filter) bool
func FilterFile(src *File, f Filter) bool
func FilterPackage(pkg *Package, f Filter) bool
func NotNilFilter(_ string, v reflect.Value) bool
func Inspect(node Node, f func(Node) bool)
func FileExports(src *File) bool
func IsExported(name string) bool
func PackageExports(pkg *Package) bool
func SortImports(fset *token.FileSet, f *File)

33. Node%methods
For$node$N$(e.g.,$N$=$Comment,$Field,$IdentExpr,$
SwitchStmt$and$so$on)
// position of first character belonging to the node
func (x *N) End() token.Pos
// position of first character immediately after the node
func (x *N) Pos() token.Pos
and$for$CommentGroup
// Text returns the text of the comment.
func (g *CommentGroup) Text() string

34. Comment'and'CommentGroup
• Node
• Expression
• Statement
• Declara4on
• Comment'and'CommentGroup
• File6and6Package

35. Comment!and!CommentGroup
// A Comment node represents a single //-style or /*-style comment.
type Comment struct {
// position of "/" starting the comment
Slash token.Pos
// comment text (excluding 'n' for //-style comments)
Text string
}
// A CommentGroup represents a sequence of comments
// with no other tokens and no empty lines between.
//
type CommentGroup struct {
List []*Comment // len(List) > 0
}

36. Expression*and*types
• Node
• Expression
• Statement
• Declara/on
• Comment1and1CommentGroup
• File1and1Package

37. Filed!and!FieldList
// A Field represents a Field declaration list in a struct type,
// a method list in an interface type, or a parameter/result
// declaration in a signature.
type Field struct {
Doc *CommentGroup // associated documentation; or nil
Names []*Ident // field/method/parameter names; or nil if anonymous field
Type Expr // field/method/parameter type
Tag *BasicLit // field tag; or nil
Comment *CommentGroup // line comments; or nil
}
// A FieldList represents a list of Fields, enclosed
// by parentheses or braces.
type FieldList struct {
Opening token.Pos // position of opening parenthesis/brace, if any
List []*Field // field list; or nil
Closing token.Pos // position of closing parenthesis/brace, if any
}

38. Expressions
type (
BadExpr struct { ... }
Ident struct { ... }
Ellipsis struct { ... }
BasicLit struct { ... }
FuncLit struct { ... }
CompositeLit struct { ... }
ParenExpr struct { ... }
SelectorExpr struct { ... }
IndexExpr struct { ... }
SliceExpr struct { ... }
TypeAssertExpr struct { ... }
CallExpr struct { .... }
StarExpr struct { ... }
UnaryExpr struct { ... }
BinaryExpr struct { ... }
KeyValueExpr struct { ... }
)

39. Types
// The direction of a channel type is indicated by one
// of the following constants.
type ChanDir int
const (
SEND ChanDir = 1 << iota
RECV
)
type (
ArrayType struct { ... }
StructType struct { ... }
FuncType struct { ... }
InterfaceType struct { ... }
MapType struct { ... }
ChanType struct { ... }
)

40. Statement
• Node
• Expression
• Statement
• Declara1on
• Comment5and5CommentGroup
• File5and5Package

41. Statements
type (
BadStmt struct { ... }
DeclStmt struct { ... }
EmptyStmt struct { ... }
LabeledStmt struct { ... }
ExprStmt struct { ... }
SendStmt struct { ... }
IncDecStmt struct { ... }
AssignStmt struct { ... }
GoStmt struct { ... }
DeferStmt struct { ... }
ReturnStmt struct { ... }
BranchStmt struct { ... }
BlockStmt struct {... }
IfStmt struct { ... }
CaseClause struct { ... }
SwitchStmt struct { ... }
TypeSwitchStmt struct { ... }
CommClause struct { ... }
SelectStmt struct { ... }
ForStmt struct { ... }
RangeStmt struct { ... }
)

42. Declara'on
• Node
• Expression
• Statement
• Declara'on
• Comment2and2CommentGroup
• File2and2Package

43. Declara'on
type (
// The Spec type stands for any of *ImportSpec,
// *ValueSpec, and *TypeSpec.
Spec interface { ... }
ImportSpec struct { ... }
ValueSpec struct { ... }
TypeSpec struct { ... }
)
type (
BadDecl struct { ... }
GenDecl struct { ... } // General declaration node
FuncDecl struct { ... }
)

44. File%and%Package
• Node
• Expression
• Statement
• Declara4on
• Comment6and6CommentGroup
• File%and%Package

45. File!and!Package
type File struct {
Doc *CommentGroup // associated documentation; or nil
Package token.Pos // position of "package" keyword
Name *Ident // package name
Decls []Decl // top-level declarations; or nil
Scope *Scope // package scope (this file only)
Imports []*ImportSpec // imports in this file
Unresolved []*Ident // unresolved identifiers in this file
Comments []*CommentGroup // list of all comments in the source file
}
type Package struct {
Name string // package name
Scope *Scope // package scope across all files
Imports map[string]*Object // map of package id -> package object
Files map[string]*File // Go source files by filename
}

46. parser
1. token
2. scanner
3. ast
4. parser

48. parser
// The parser structure holds the parser's internal state.
type parser struct {
file *token.File
errors scanner.ErrorList
scanner scanner.Scanner
...
// Next token
pos token.Pos // token position
tok token.Token // one token look-ahead
lit string // token literal
...
// Ordinary identifier scopes
pkgScope *ast.Scope // pkgScope.Outer == nil
topScope *ast.Scope // top-most scope; may be pkgScope
unresolved []*ast.Ident // unresolved identifiers
imports []*ast.ImportSpec // list of imports
// Label scopes
// (maintained by open/close LabelScope)
labelScope *ast.Scope // label scope for current function
targetStack [][]*ast.Ident // stack of unresolved labels
}

49. Parsing(func,ons
func ParseFile(fset *token.FileSet, filename string,
src interface{}, mode Mode) (f *ast.File, err error)
func ParseDir(fset *token.FileSet, path string,
filter func(os.FileInfo) bool,
mode Mode) (pkgs map[string]*ast.Package, first error)
// ParseExpr is a convenience function for obtaining
// the AST of an expression x.
func ParseExpr(x string) (ast.Expr, error)

50. Mode
type Mode uint
const (
// stop parsing after package clause
PackageClauseOnly Mode = 1 << iota
// stop parsing after import declarations
ImportsOnly
// parse comments and add them to AST
ParseComments
// print a trace of parsed productions
Trace
// report declaration errors
DeclarationErrors
// same as AllErrors, for backward-compatibility
SpuriousErrors
// report all errors (not just the first 10 on different lines)
AllErrors = SpuriousErrors
)

51. Usage&example&in&
example_test.go
var fset = token.NewFileSet()
var validFiles = []string{
"parser.go",
"parser_test.go",
"error_test.go",
"short_test.go",
}
func TestParse(t *testing.T) {
for _, filename := range validFiles {
_, err := ParseFile(fset, filename, nil, DeclarationErrors)
if err != nil {
t.Fatalf("ParseFile(%s): %v", filename, err)
}
}
}

52. Usage&example&in&
example_test.go
// This example shows what an AST looks like when printed for debugging.
func ExamplePrint() {
// src is the input for which we want to print the AST.
src := `
package main
func main() {
println("Hello, World!")
}
`
// Create the AST by parsing src.
fset := token.NewFileSet() // positions are relative to fset
f, err := parser.ParseFile(fset, "", src, 0)
if err != nil {
panic(err)
}
// Print the AST.
ast.Print(fset, f)
}

53. U"li"es

54. U"li"es
1. printer
2. doc
3. format
4. build

55. printer
1. printer
2. doc
3. format
4. build

56. Fprint
// Fprint "pretty-prints" an AST node to output.
// It calls Config.Fprint with default settings.
func Fprint(output io.Writer,
fset *token.FileSet,
node interface{}) error

57. Config!and!Mode
// A Config node controls the output of Fprint.
type Config struct {
Mode Mode // default: 0
Tabwidth int // default: 8
// default: 0 (all code is indented at least by this much)
Indent int
}
func (cfg *Config) Fprint(output io.Writer, fset *token.FileSet,
node interface{}) error
// A Mode value is a set of flags (or 0). They control printing.
type Mode uint
const (
// do not use a tabwriter; if set, UseSpaces is ignored
RawFormat Mode = 1 << iota
// use tabs for indentation independent of UseSpaces
TabIndent
// use spaces instead of tabs for alignment
UseSpaces
// emit //line comments to preserve original source positions
SourcePos
)

58. CommentedNode
// A CommentedNode bundles an AST node and corresponding comments.
// It may be provided as argument to any of the Fprint functions.
type CommentedNode struct {
// *ast.File, or ast.Expr, ast.Decl, ast.Spec, or ast.Stmt
Node interface{}
Comments []*ast.CommentGroup
}

59. doc
1. printer
2. doc
3. format
4. build

60. doc
Types&of&doc
type Func struct { ... } // func foo() { ... }
type Note struct { ... } // TODO(tfukushima): ...
type Package struct { ... } // package bar
type Type struct { ... } // type
type Value struct { ... } // var, const

62. Package!methods
func New(pkg *ast.Package, importPath string, mode Mode) *Package
func (p *Package) Filter(f Filter)
type Filter func(string) bool
// Mode values control the operation of New.
type Mode int
const (
// extract documentation for all package-level
// declarations, not just exported ones
AllDecls Mode = 1 << iota
// show all embedded methods, not just the ones of
// invisible (unexported) anonymous fields
AllMethods
)

63. Every&exported&(capitalized)&name&in&a&
program&should&have&a&doc&comment
Python
def _this_is_an_unexported_function():
...
def this_is_an_exported_function():
"""This is a Docstring. Lisp has it, too.
"""
Go
func thisIsAnUnexportedFunction() { ... }
// This should have a doc comment.
func ThisIsAnExportedFunction() { ... }

64. Example
// An Example represents an example function found in a source files.
type Example struct {
Name string // name of the item being exemplified
Doc string // example function doc string
Code ast.Node
Play *ast.File // a whole program version of the example
Comments []*ast.CommentGroup
Output string // expected output
EmptyOutput bool // expect empty output
Order int // original source code order
}
// Examples returns the examples found in the files, sorted by
// Name field.
func Examples(files ...*ast.File) []*Example

65. Synopsis
// Synopsis returns a cleaned version of the first sentence in s.
func Synopsis(s string) string
var IllegalPrefixes = []string{
"copyright",
"all rights",
"author",
}

66. ToHTML!and!ToText
// ToHTML converts comment text to formatted HTML.
func ToHTML(w io.Writer, text string,
words map[string]string)
// ToText prepares comment text for presentation in textual
// output.
func ToText(w io.Writer, text string, indent,
preIndent string, width int)

67. format
1. printer
2. doc
3. format
4. build

68. Node!and!Source
// Node formats node in canonical gofmt style and writes the result
// to dst.
func Node(dst io.Writer, fset *token.FileSet, node interface{}) error
// Source formats src in canonical gofmt style and returns the result
// or an (I/O or syntax) error.
func Source(src []byte) ([]byte, error)

69. Usage&Example:&My&go fmt
import (
"go/format"
"io/ioutil"
"os"
)
const fileName = "hello.go"
func formatFile(fileName string) {
src, err := ioutil.ReadFile(fileName)
if err != nil { panic(err) }
res, err := format.Source(src)
if err != nil { panic(err) }
ioutil.WriteFile(fileName, res, os.ModeType)
}
func main() {
formatFile(fileName)
}

70. Resulf'of'the'usage'example
/Users/tfukushima/go/src/github.com/tfukushima/mygofmt% cat hello.go
package main
import ("fmt"
"github.com/tfukushima/string")
func
main(){fmt.Println(string.Reverse("Hello, new gopher!"))}
/Users/tfukushima/go/src/github.com/tfukushima/mygofmt% go run mygofmt.go
/Users/tfukushima/go/src/github.com/tfukushima/mygofmt% cat hello.go
package main
import (
"fmt"
"github.com/tfukushima/string"
)
func main() {
fmt.Println(string.Reverse("Hello, new gopher!"))
}

71. build
1. printer
2. doc
3. format
4. build

72. Go#Path
GOPATH=/home/user/gocode
/home/user/gocode/
src/
foo/
bar/ (go code in package bar)
x.go
quux/ (go code in package main)
y.go
bin/
quux (installed command)
pkg/
linux_amd64/
foo/
bar.a (installed package object)

73. Build&constraints
To#build#only#on#the#specific#OS#and/or#arch:
// +build linux,386 darwin,!cgo
(linux AND 386) OR (darwin AND (NOT cgo))
// +build linux darwin
// +build 386
(linux OR darwin) AND 386
To#keep#a#file#from#being#considered#for#the#build:
// +build ignore
// +build unsatisfied_word

74. Build&constraints
During'a'par*cular'build,'the'following'words'are'
sa*sfied:
- the target operating system, as spelled by runtime.GOOS
- the target architecture, as spelled by runtime.GOARCH
- the compiler being used, either "gc" or "gccgo"
- "cgo", if ctxt.CgoEnabled is true
- "go1.1", from Go version 1.1 onward
- "go1.2", from Go version 1.2 onward
- "go1.3", from Go version 1.3 onward
- any additional words listed in ctxt.BuildTags

75. Build&constraints&by&file&names
- source_windows_arm64.go
- source_windows_arm64_test.go
GOOS=windows
GOARCH=arm64

76. Package
// A Package describes the Go package found in a directory.
type Package struct {
Dir string // directory containing package sources
Name string // package name
Doc string // documentation synopsis
ImportPath string // import path of package ("" if unknown)
Root string // root of Go tree where this package lives
SrcRoot string // package source root directory ("" if unknown)
PkgRoot string // package install root directory ("" if unknown)
BinDir string // command install directory ("" if unknown)
Goroot bool // package found in Go root
PkgObj string // installed .a file
AllTags []string // tags that can influence file selection in this directory
ConflictDir string // this directory shadows Dir in $GOPATH
...
}
// IsCommand reports whether the package is considered a command to be installed
// (not just a library). Packages named "main" are treated as commands.
func (p *Package) IsCommand() bool {
return p.Name == "main"
}

77. Context
// A Context specifies the supporting context for a build.
type Context struct {
GOARCH string // target architecture
GOOS string // target operating system
GOROOT string // Go root
GOPATH string // Go path
CgoEnabled bool // whether cgo can be used
UseAllFiles bool // use files regardless of +build lines, file names
Compiler string // compiler to assume when computing target paths
...
}
// Default is the default Context for builds. It uses the GOARCH, GOOS,
// GOROOT, and GOPATH environment variables if set, or else the compiled
// code's GOARCH, GOOS, and GOROOT.
var Default Context = defaultContext()

78. Context!methods
func (ctxt *Context) Import(path string, srcDir string,
mode ImportMode) (*Package, error)
func (ctxt *Context) ImportDir(dir string,
mode ImportMode) (*Package, error)
func (ctxt *Context) MatchFile(dir, name string) (match bool, err error)
// SrcDirs returns a list of package source root directories. It
// draws from the current Go root and Go path but omits directories
// that do not exist.
func (ctxt *Context) SrcDirs() []string

79. ImportMode
type ImportMode uint
const (
// If FindOnly is set, Import stops after locating the directory
// that should contain the sources for a package. It does not
// read any files in the directory.
FindOnly ImportMode = 1 << iota
// If AllowBinary is set, Import can be satisfied by a compiled
// package object without corresponding sources.
AllowBinary
)

80. NoGoError
// NoGoError is the error used by Import to describe a directory
// containing no buildable Go source files. (It may still contain
// test files, files hidden by build tags, and so on.)
type NoGoError struct {
Dir string
}
func (e *NoGoError) Error() string {
return "no buildable Go source files in " + e.Dir
}

81. Import!and!ImportDir
// Import is shorthand for Default.Import.
func Import(path, srcDir string, mode ImportMode) (*Package, error)
// Import is shorthand for Default.ImportDir.
func ImportDir(dir string, mode ImportMode) (*Package, error)

82. Misc
// ToolDir is the directory containing build tools.
var ToolDir = filepath.Join(runtime.GOROOT(),
"pkg/tool/"+runtime.GOOS+"_"+runtime.GOARCH)
// ArchChar returns the architecture character for the given goarch.
// For example, ArchChar("amd64") returns "6".
func ArchChar(goarch string) (string, error)
// IsLocalImport reports whether the import path is
// a local import path, like ".", "..", "./foo", or "../foo".
func IsLocalImport(path string) bool

83. Wrap%up

84. go!package!hierarchy
• ast
• build
• doc
• format
• parser
• printer
• scanner
• token

85. go!package:!Sta+c!analysis
• ast
• build
• doc
• format
• parser
• printer
• scanner
• token

86. go!package:!U*li*es
• ast
• build
• doc
• format
• parser
• printer
• scanner
• token

87. The$end$of$slides.$Any$
ques1on?