- TMU Komachi lab - paper reading - How Contextual are Contextualized Word Representations? Kawin Ethayarajh. EMNLP2019 - paper URL: https://arxiv.org/abs/1909.00512

2.
!2

3.
!3

4.
!4
w {s1, . . . , sn}
{i1, . . . , in}
w = s1[i1] = . . . = sn[in]
fl(s, i) s[i]
l
SelfSiml(w) =
1
n2 − n ∑
j
∑
k≠j
cos
(
fl (sj, ij), fl (sk, ik))

5.
!5
s ⟨w1, . . . , wn⟩
fl(s, i) s[i]
l
IntraSiml(s) =
1
n ∑
i
cos ( ⃗sl, fl(s, i))
where ⃗sl =
1
n ∑
i
fl(s, i)

6.
!6
w {s1, . . . , sn}
{i1, . . . , in}
w = s1[i1] = . . . = sn[in]
fl(s, i) s[i] l
MEVl(w) =
σ2
1
∑i
σ2
i
[fl(s1, i1) . . . fl(sn, in)]
σ1 . . . σm

7.
!7
Baseline (fl) = 𝔼x,y∼U(𝒪) [cos (fl(x), fl(y))]
𝒪
fl( ⋅ )
l

8.
!8
Baseline (fl) = 𝔼x,y∼U(𝒪) [cos (fl(x), fl(y))]
𝒪
fl( ⋅ )
SelfSiml(w) = 0.95
Baseline(fl) = 0.00
Baseline(fl) = 0.99
Baseline(fl) = 0.00
Baseline(fl) = 0.99

9.
!9
Baseline (fl) = 𝔼x,y∼U(𝒪) [cos (fl(x), fl(y))]
𝒪
fl( ⋅ )
SelfSiml(w) = 0.95
Baseline(fl) = 0.00
Baseline(fl) = 0.99
Baseline(fl) = 0.00
Baseline(fl) = 0.99

10.
!10
Baseline (fl) = 𝔼x,y∼U(𝒪) [cos (fl(x), fl(y))]
𝒪
fl( ⋅ )
Baseline (fl) = 𝔼x,y∼U(𝒪) [cos (fl(x), fl(y))]
SelfSim*l
(w) = SelfSiml(w) − Baseline (fl)

11. !11
Baseline (fl) = 𝔼x,y∼U(𝒪) [cos (fl(x), fl(y))]

12. !12
Baseline (fl) = 𝔼x,y∼U(𝒪) [cos (fl(x), fl(y))]

13. !13
Baseline (fl) = 𝔼x,y∼U(𝒪) [cos (fl(x), fl(y))]

14.
!14
SelfSiml(w) =
1
n2 − n ∑
j
∑
k≠j
cos
(
fl (sj, ij), fl (sk, ik))

15.
!15
SelfSiml(w) =
1
n2 − n ∑
j
∑
k≠j
cos
(
fl (sj, ij), fl (sk, ik))

16.
!16
SelfSiml(w) =
1
n2 − n ∑
j
∑
k≠j
cos
(
fl (sj, ij), fl (sk, ik))

17.
!17
IntraSiml(s) =
1
n ∑
i
cos ( ⃗sl, fl(s, i))
where ⃗sl =
1
n ∑
i
fl(s, i)

18.
!18
IntraSiml(s) =
1
n ∑
i
cos ( ⃗sl, fl(s, i))
where ⃗sl =
1
n ∑
i
fl(s, i)

19.
!19
MEVl(w) =
σ2
1
∑i
σ2
i

20.
!20
MEVl(w) =
σ2
1
∑i
σ2
i

21.
!21
MEVl(w) =
σ2
1
∑i
σ2
i

22.
!22

23.
!23

24. !24