-
Notifications
You must be signed in to change notification settings - Fork 1
/
Copy pathLM.hs
266 lines (254 loc) · 13.1 KB
/
LM.hs
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
{-# LANGUAGE PartialTypeSignatures #-}
{-# LANGUAGE ConstraintKinds #-}
{-# LANGUAGE FlexibleContexts #-}
{-# LANGUAGE ViewPatterns #-}
{-# LANGUAGE AllowAmbiguousTypes #-}
{-# OPTIONS_GHC -fplugin GHC.TypeLits.KnownNat.Solver #-}
{-# LANGUAGE DataKinds #-}
{-# LANGUAGE RankNTypes #-}
{-# LANGUAGE ScopedTypeVariables #-}
{-# LANGUAGE TypeApplications #-}
{-# LANGUAGE TypeFamilies #-}
{-# LANGUAGE TypeOperators #-}
{-# LANGUAGE UnicodeSyntax #-}
{-# LANGUAGE RankNTypes #-}
{-# LANGUAGE NoStarIsType #-}
import TypedFlow.Layers.RNN.Base
import TypedFlow.TF
import TypedFlow
import TypedFlow.Types
import TypedFlow.Types.Proofs
import GHC.TypeLits
import TypedFlow.Layers.Core (DenseP(..),(#))
import Prelude hiding (RealFrac(..))
import TypedFlow.Python
import OrnUtils
import XOrn (mkXORN)
import FFTOrn (mkFORN, type Pow2, type OrnAnglesShape, type OrnEmbSize, knownPowTwo, knownOrnAnglesShape)
import LSTMUtils
import System.Environment
lstmModel :: forall (len :: Nat) (vocSize::Nat). KnownNat vocSize => KnownNat len =>
Gen [Function]
lstmModel = do
-- dropw <- mkDropout (DropProb 0.95)
embs <- parameterDefault "embs"
-- wstk <- parameterDefault "stk"
let dropProb = DropProb 0.10
-- lstm1 <- mkLSTM @40 "w1" dropProb
-- gru1 <- mkGRU @160 "w1" dropProb
lstm1 <- mkLSTM @160 "w1" dropProb
-- lstm2 <- mkLSTM @160 "w2" dropProb
-- lstm3 <- mkLSTM @160 "w3" dropProb
-- lstm4 <- mkLSTM @160 "w4" dropProb
-- lstm2 <- mkLSTM @160 "w2" dropProb
drp <- mkDropout dropProb
w <- parameterDefault "dense"
let lm :: (Placeholders '[ '("x",'[len],Int32),
'("y",'[len],Int32),
'("weights",'[len],Float32)]) ->
ModelOutput Float32 '[len,vocSize] '[]
lm (PHT input :* PHT gold :* PHT masks :* Unit) =
let (_sFi,predictions)
= simpleRnn (timeDistribute (embedding @12 @vocSize embs) .-.
-- gru1 .-.
lstm1 .-.
-- lstm2.-. lstm3.-. lstm4 .-.
-- (stackCell @12 wstk) .-.
-- withFeedback (lstm1 .-. (stackCell @8 wstk)) .-.
timeDistribute drp .-.
timeDistribute (dense w))
(repeatT zeros,input)
in timedCategorical masks predictions gold
return [modelFunction "runModel" lm]
urnModel :: forall (len :: Nat) (vocSize::Nat) units embSize stateShape.
(units ~ 32, embSize ~ 90) => -- 3 bands + 2
-- (units ~ 32, embSize ~ 88) => -- 3 bands
-- (units ~ 50, embSize ~ (49 + 48 + 47)) => -- 3 bands
-- (units ~ 50, embSize ~ (25*49)) => -- 3 full
-- (units ~ 32, l ~ 496) => -- full
(stateShape ~ '[ '[units]]) =>
KnownNat vocSize => KnownNat len =>
Gen [Function]
urnModel = do
embs@(EmbeddingP embMat) <- parameterDefault "embs" -- embedding layer
let dropProb = DropProb 0.05
theRNN <- mkD @embSize @units dropProb
drp <- mkDropout dropProb
drp1 <- mkDropout dropProb
-- w <- parameter "projection" glorotUniform -- projection layer
proj <- parameterDefault "projection"
let initialState :: HTV Float32 stateShape
initialState = VecSing (oneHot0 @units @'B32 (constant 0)) -- start with [1 0 0 0 ... ] vector
-- projectionLayer = timeDistribute (w ∙)
projectionLayer = timeDistribute (proj #)
base = timeDistribute (embedding @embSize @vocSize embs) .-.
timeDistribute drp1 .-.
theRNN .-.
timeDistribute drp
run :: KnownTyp t1 => KnownShape s1 => KnownShape s0 => KnownNat n => (t2 ~ Float32)
=> RnnCell t2 stateShape (T s1 t1) (T s0 t0)
-> (Tensor (n : s1) t1)
-> (Tensor (n : s0) t0)
run cell input = snd $ simpleRnn cell (initialState ,input)
net = base .-. projectionLayer
lm :: Placeholders '[ '("x",'[len],Int32), '("y",'[len],Int32), '("weights",'[len],Float32)] -> ModelOutput Float32 '[len,vocSize] '[]
lm (PHT input :* PHT gold :* PHT masks :* Unit) = timedCategorical masks (run net input) gold
probeStates :: Placeholders '[ '("x",'[len],Int32) ] -> Placeholders '[ '("states", '[len,units], Float32) ]
probeStates (PHT input :* Unit) = PHT ((run base) input) :* Unit
probeEmbs :: Placeholders '[ '("wordIdx", '[], Int32) ]
-> Placeholders '[ '("embsAntiHermitian", '[units,units], Float32) ]
probeEmbs (PHT idx :* Unit) = PHT (embToAntiHermitian @embSize (lookupT idx embMat)) :* Unit
probePreds :: Placeholders '[ '("x",'[len],Int32) ] -> Placeholders '[ '("pred" , '[len,vocSize], Float32), '("y" , '[len], Int32)]
probePreds (PHT input :* Unit) = let ps = run net input
in PHT ps :* PHT (mapT argmax0 ps) :* Unit
return [modelFunction "runModel" lm
,probeFunction "probeStates" probeStates
,probeFunction "probePreds" probePreds
,probeFunction "probeEmbs" probeEmbs]
furnModel :: forall (len :: Nat) (vocSize::Nat) units embSize stateShape.
(stateShape ~ '[ '[ Pow2 units] ]
,embSize ~ OrnEmbSize units
,KnownPeano units
-- ,units ~ 'Succ ('Succ ('Succ 'Zero))
,units ~ 'Succ ('Succ ('Succ ('Succ ('Succ 'Zero)))) -- attn: 2^...
) =>
KnownNat vocSize => KnownNat len =>
Gen [Function]
furnModel = knownPowTwo @units ?>
knownOrnAnglesShape (knownPeano @units) ?>
knownProduct @(OrnAnglesShape units) ?>
do
initState <- normalize <$> parameter "initstate" (noise (UniformD (-1) 1)) -- can't use the 1 0 0 0 state because the network is not isotropic!
embMat <- parameter "embs" (noise (UniformD (-pi/3) (pi/3)))
let embs = EmbeddingP embMat -- <- parameterDefault "embs" -- embedding layer
let dropProb = DropProb 0
theRNN <- mkFORN @units dropProb
drp <- mkDropout dropProb
drp1 <- mkDropout dropProb
-- w <- parameter "projection" glorotUniform -- projection layer
proj <- parameterDefault "projection"
let initialState :: HTV Float32 stateShape
initialState = VecSing initState
-- projectionLayer = timeDistribute (w ∙)
projectionLayer = timeDistribute (proj #)
base = timeDistribute (embedding @(Product (OrnAnglesShape units)) @vocSize embs) .-.
timeDistribute drp1 .-.
theRNN .-.
timeDistribute drp
run :: KnownTyp t1 => KnownShape s1 => KnownShape s0 => KnownNat n => (t2 ~ Float32)
=> RnnCell t2 stateShape (T s1 t1) (T s0 t0)
-> (Tensor (n : s1) t1)
-> (Tensor (n : s0) t0)
run cell input = snd $ simpleRnn cell (initialState ,input)
net = base .-. projectionLayer
lm :: Placeholders '[ '("x",'[len],Int32), '("y",'[len],Int32), '("weights",'[len],Float32)] -> ModelOutput Float32 '[len,vocSize] '[]
lm (PHT input :* PHT gold :* PHT masks :* Unit) = timedCategorical masks (run net input) gold
-- probeStates :: Placeholders '[ '("x",'[len],Int32) ] -> Placeholders '[ '("states", '[len,units], Float32) ]
-- probeStates (PHT input :* Unit) = PHT ((run base) input) :* Unit
-- probeEmbs :: Placeholders '[ '("wordIdx", '[], Int32) ]
-- -> Placeholders '[ '("embsAntiHermitian", '[units,units], Float32) ]
-- probeEmbs (PHT idx :* Unit) = PHT (embToAntiHermitian @embSize (lookupT idx embMat)) :* Unit
-- probePreds :: Placeholders '[ '("x",'[len],Int32) ] -> Placeholders '[ '("pred" , '[len,vocSize], Float32), '("y" , '[len], Int32)]
-- probePreds (PHT input :* Unit) = let ps = run net input
-- in PHT ps :* PHT (mapT argmax0 ps) :* Unit
return [modelFunction "runModel" lm
-- ,probeFunction "probeStates" probeStates
-- ,probeFunction "probePreds" probePreds
-- ,probeFunction "probeEmbs" probeEmbs
]
xurnModel :: forall (len :: Nat) (vocSize::Nat) n l embShape stateShape.
(embShape ~ '[ l * n ]
,stateShape ~ '[ '[n*2 + 1] ]
,KnownNat n
,KnownNat l
-- , n ~ 31
-- ,l ~ 3
)
=> KnownNat vocSize => KnownNat len
=> Gen [Function]
xurnModel =
do
embMat <- parameter "embs" (noise (UniformD (-pi/3) (pi/3)))
let embs = (EmbeddingP @vocSize embMat) -- <- parameterDefault "embs" -- embedding layer
let dropProb = DropProb 0.05
(theRNN :: (RnnCell t '[ '[n*2 + 1] ] (Tensor '[l*n] t) (Tensor '[n*2 + 1] t))) <- mkXORN @l @n dropProb
drp <- mkDropout dropProb
drp1 <- mkDropout dropProb
-- w <- parameter "projection" glorotUniform -- projection layer
proj <- parameterDefault "projection"
let initialState :: HTV Float32 stateShape
initialState = VecSing (oneHot0 @(n*2+1) @'B32 (constant 0)) -- start with [1 0 0 0 ... ] vector
-- projectionLayer = timeDistribute (w ∙)
projectionLayer = timeDistribute (proj #)
base = timeDistribute (embedding @(l*n) @vocSize embs) .-.
timeDistribute drp1 .-.
theRNN .-.
timeDistribute drp
run :: KnownTyp t1 => KnownShape s1 => KnownShape s0 => KnownNat len => (t2 ~ Float32)
=> RnnCell t2 stateShape (T s1 t1) (T s0 t0) -> (Tensor (len : s1) t1) -> (Tensor (len : s0) t0)
run cell input = snd $ simpleRnn cell (initialState ,input)
net = base .-. projectionLayer
lm :: Placeholders '[ '("x",'[len],Int32), '("y",'[len],Int32), '("weights",'[len],Float32)] -> ModelOutput Float32 '[len,vocSize] '[]
lm (PHT input :* PHT gold :* PHT masks :* Unit) = timedCategorical masks (run net input) gold
-- probeStates :: Placeholders '[ '("x",'[len],Int32) ] -> Placeholders '[ '("states", '[len,units], Float32) ]
-- probeStates (PHT input :* Unit) = PHT ((run base) input) :* Unit
-- probePreds :: Placeholders '[ '("x",'[len],Int32) ] -> Placeholders '[ '("pred" , '[len,vocSize], Float32), '("y" , '[len], Int32)]
-- probePreds (PHT input :* Unit) = let ps = run net input
-- in PHT ps :* PHT (mapT argmax0 ps) :* Unit
return [modelFunction "runModel" lm
-- ,probeFunction "probeStates" probeStates
-- ,probeFunction "probePreds" probePreds
-- ,probeFunction "probeEmbs" probeEmbs
]
mulModel :: forall (len :: Nat) (vocSize::Nat) units stateShape embSize.
(units ~ 50) =>
(embSize ~ (units * units)) =>
-- (units ~ 32, l ~ 496) => -- full
(stateShape ~ '[ '[units]]) =>
KnownNat vocSize => KnownNat len =>
Gen [Function]
mulModel = do
embs <- parameterDefault "embs" -- embedding layer
let dropProb = DropProb 0.05
theRNN <- mkMul @units dropProb
drp <- mkDropout dropProb
drp1 <- mkDropout dropProb
-- w <- parameter "projection" glorotUniform -- projection layer
proj <- parameterDefault "projection"
let initialState :: HTV Float32 stateShape
initialState = VecSing (oneHot0 @units @'B32 (constant 0)) -- start with [1 0 0 0 ... ] vector
-- projectionLayer = timeDistribute (w ∙)
projectionLayer = timeDistribute (proj #)
base = timeDistribute (embedding @embSize @vocSize embs) .-.
timeDistribute drp1 .-.
theRNN .-.
timeDistribute drp
run :: KnownTyp t1 => KnownShape s1 => KnownShape s0 => KnownNat n => (t2 ~ Float32)
=> RnnCell t2 stateShape (T s1 t1) (T s0 t0)
-> (Tensor (n : s1) t1)
-> (Tensor (n : s0) t0)
run cell input = snd $ simpleRnn cell (initialState ,input)
net = base .-. projectionLayer
lm :: Placeholders '[ '("x",'[len],Int32), '("y",'[len],Int32), '("weights",'[len],Float32)] -> ModelOutput Float32 '[len,vocSize] '[]
lm (PHT input :* PHT gold :* PHT masks :* Unit) = timedCategorical masks (run net input) gold
probeStates :: Placeholders '[ '("x",'[len],Int32) ] -> Placeholders '[ '("states", '[len,units], Float32) ]
probeStates (PHT input :* Unit) = PHT ((run base) input) :* Unit
probePreds :: Placeholders '[ '("x",'[len],Int32) ] -> Placeholders '[ '("pred" , '[len,vocSize], Float32), '("y" , '[len], Int32)]
probePreds (PHT input :* Unit) = let ps = run net input
in PHT ps :* PHT (mapT argmax0 ps) :* Unit
return [modelFunction "runModel" lm
,probeFunction "probeStates" probeStates
,probeFunction "probePreds" probePreds]
main :: IO ()
main = do
[modelKind] <- getArgs
let targetModel :: forall (len :: Nat) (vocSize::Nat). KnownNat len => KnownNat vocSize => Gen [Function]
targetModel = case modelKind of
"forn" -> furnModel @len @vocSize
"orn" -> urnModel @len @vocSize
"lstm" -> lstmModel @len @vocSize
"mul" -> mulModel @len @vocSize
"xorn" -> xurnModel @len @vocSize @32 @5
_ -> error "main: unknown modelKind"
generateFile (modelKind <> "_lm.py") (compileGen @512 defaultOptions (targetModel @21 @12))
putStrLn "done!"