Theano学习-scan循环
\(1.Scan\)
- 通用的一般形式,可用于循环
- 减少和映射(对维数循环)是特殊的 \(scan\)
- 对输入序列进行 \(scan\) 操作,每一步都能得到一个输出
- \(scan\) 能看到定义函数的前 \(k\) 个时间结果
- 给定初始值 \(z=0\), \(sum()\) 函数能扫描计算列表的和 \(z+x(i)\)
- 通常一个 \(for\) 循环能用 \(scan()\) 表示,\(scan\) 是最接近 \(Theano\) 中表示循环的方式
- 使用 \(scan\) 表示循环的好处:
- 迭代数成为了符号图的一部分
- \(GPU\) 传输最少(假如用了 \(GPU\) )
- 执行比编译好的使用 \(Python\) 的 \(for\) 循环快一些
- 通过检测到的实际内存量降低整体内存使用量
\(Scan\) 例子:计算 \(tanh(x(t).dot(W) + b)\)
import theano
import theano.tensor as T
import numpy as np
X = T.matrix("X")
W = T.matrix("W")
b_sym = T.vector("b_sym")
results, updates = theano.scan(lambda v: T.tanh(T.dot(v, W) + b_sym), sequences = X)
compute_elementwise = theano.function(inputs = [X, W, b_sym], outputs = results)
x = np.eye(2, dtype = theano.config.floatX)
w = np.ones((2, 2), dtype = theano.config.floatX)
b = np.ones((2), dtype = theano.config.floatX)
b[1] = 2
print('x: ', x)
print('w: ', w)
print('b: ', b)
print(compute_elementwise(x, w, b))
print(np.tanh(x.dot(w) + b))
#####
print
x: [[ 1. 0.]
[ 0. 1.]]
w: [[ 1. 1.]
[ 1. 1.]]
b: [ 1. 2.]
[[ 0.96402758 0.99505475]
[ 0.96402758 0.99505475]]
[[ 0.96402758 0.99505475]
[ 0.96402758 0.99505475]]
#####
\(Scan\) 例子:计算序列 \(x(t)=tanh(x(t-1).dot(W)+y(t).dot(U)+p(T-t).dot(V))\)
import theano
import theano.tensor as T
import numpy as np
X = T.vector('X')
W = T.matrix('W')
b_sym = T.vector('b_sym')
U = T.matrix('U')
Y = T.matrix('Y')
V = T.matrix('V')
P = T.matrix('P')
results, updates = theano.scan(lambda y, p, x_tml: T.tanh(T.dot(x_tml, W) + T.dot(y, U) + T.dot(p, V)), sequences = [Y, P[::-1]], outputs_info = [X])
compute_seq = theano.function(inputs = [X, W, Y, U, P, V], outputs = results)
x = np.zeros((2), dtype = theano.config.floatX)
x[1] = 1
w = np.ones((2, 2), dtype = theano.config.floatX)
y = np.ones((5, 2), dtype = theano.config.floatX)
y[0, :] = -3
u = np.ones((2, 2), dtype = theano.config.floatX)
p = np.ones((5, 2), dtype = theano.config.floatX)
p[0, :] = 3
v = np.ones((2, 2), dtype = theano.config.floatX)
print(compute_seq(x, w, y, u, p, v))
x_res = np.zeros((5, 2), dtype = theano.config.floatX)
x_res[0] = np.tanh(x.dot(w) + y[0].dot(u) + p[4].dot(v))
for i in range(1, 5):
x_res[i] = np.tanh(x_res[i - 1].dot(w) + y[i].dot(u) + p[4- i].dot(v))
print(x_res)
#####print
[[-0.99505475 -0.99505475]
[ 0.96471973 0.96471973]
[ 0.99998585 0.99998585]
[ 0.99998771 0.99998771]
[ 1. 1. ]]
[[-0.99505475 -0.99505475]
[ 0.96471973 0.96471973]
[ 0.99998585 0.99998585]
[ 0.99998771 0.99998771]
[ 1. 1. ]]
#####
\(Scan\) 例子:计算 \(X\) 的行范数
import theano
import theano.tensor as T
import numpy as np
X = T.matrix("X")
results, updates = theano.scan(lambda x_i: T.sqrt((x_i ** 2).sum()), sequences = [X])
compute_norm_lines = theano.function(inputs = [X], outputs = results)
x = np.diag(np.arange(1, 6, dtype = theano.config.floatX), 1)
print(compute_norm_lines(x))
print(np.sqrt((x ** 2).sum(1)))
#####print
[ 1. 2. 3. 4. 5. 0.]
[ 1. 2. 3. 4. 5. 0.]
#####
\(Scan\) 例子:计算 \(X\) 的列范数
import theano
import theano.tensor as T
import numpy as np
X = T.matrix("X")
results, updates = theano.scan(lambda x_i: T.sqrt((x_i ** 2).sum()), sequences = [X.T])
compute_norm_cols = theano.function(inputs = [X], outputs = results)
x = np.diag(np.arange(1, 6, dtype = theano.config.floatX), 1)
print(compute_norm_cols(x))
print(np.sqrt((x ** 2).sum(0)))
#####print
[ 0. 1. 2. 3. 4. 5.]
[ 0. 1. 2. 3. 4. 5.]
#####
\(Scan\) 例子:计算 \(X\) 的迹
import theano
import theano.tensor as T
import numpy as np
floatX = 'float32'
X = T.matrix("X")
results, updates = theano.scan(lambda i, j, t_f: T.cast(X[i, j] + t_f, floatX), sequences = [T.arange(X.shape[0]), T.arange(X.shape[1])], outputs_info = np.asarray(0, dtype = floatX))
result = results[-1]
compute_trace = theano.function(inputs = [X], outputs = result)
x = np.eye(5, dtype = theano.config.floatX)
x[0] = np.arange(5, dtype = theano.config.floatX)
print(compute_trace(x))
print(np.diagonal(x).sum())
#####print
4.0
4.0
#####
**\(Scan\) 例子:计算序列 \(x(t)=x(t-2).dot(U)+x(t-1).dot(V)+tanh(x(t-1).dot(W)+b)\) **
import theano
import theano.tensor as T
import numpy as np
X = T.matrix("X")
W = T.matrix("W")
b_sym = T.vector("b_sym")
U = T.matrix("U")
V = T.matrix("V")
n_sym = T.iscalar("n_sym")
results, updates = theano.scan(lambda x_tm2, x_tm1: T.dot(x_tm2, U) + T.dot(x_tm1, V) + T.tanh(T.dot(x_tm1, W) + b_sym), n_steps=n_sym, outputs_info=[dict(initial=X, taps=[-2, -1])])
compute_seq2 = theano.function(inputs=[X, U, V, W, b_sym, n_sym], outputs=results)
x = np.zeros((2, 2), dtype=theano.config.floatX)
x[1, 1] = 1
w = 0.5 * np.ones((2, 2), dtype=theano.config.floatX)
u = 0.5 * (np.ones((2, 2), dtype=theano.config.floatX) - np.eye(2, dtype=theano.config.floatX))
v = 0.5 * np.ones((2, 2), dtype=theano.config.floatX)
n = 10
b = np.ones((2), dtype=theano.config.floatX)
print(compute_seq2(x, u, v, w, b, n))
x_res = np.zeros((10, 2))
x_res[0] = x[0].dot(u) + x[1].dot(v) + np.tanh(x[1].dot(w) + b)
x_res[1] = x[1].dot(u) + x_res[0].dot(v) + np.tanh(x_res[0].dot(w) + b)
x_res[2] = x_res[0].dot(u) + x_res[1].dot(v) + np.tanh(x_res[1].dot(w) + b)
for i in range(2, 10):
x_res[i] = (x_res[i - 2].dot(u) + x_res[i - 1].dot(v) + np.tanh(x_res[i - 1].dot(w) + b))
print(x_res)
#####print
[[ 1.40514825 1.40514825]
[ 2.88898899 2.38898899]
[ 4.34018291 4.34018291]
[ 6.53463142 6.78463142]
[ 9.82972243 9.82972243]
[ 14.22203814 14.09703814]
[ 20.07439936 20.07439936]
[ 28.12291843 28.18541843]
[ 39.1913681 39.1913681 ]
[ 54.28407732 54.25282732]]
[[ 1.40514825 1.40514825]
[ 2.88898899 2.38898899]
[ 4.34018291 4.34018291]
[ 6.53463142 6.78463142]
[ 9.82972243 9.82972243]
[ 14.22203814 14.09703814]
[ 20.07439936 20.07439936]
[ 28.12291843 28.18541843]
[ 39.1913681 39.1913681 ]
[ 54.28407732 54.25282732]]
#####
\(Scan\) 例子:计算 \(y=tanh(v.dot(A))\) 的 \(Jacobian\)
import theano
import theano.tensor as T
import numpy as np
v = T.vector()
A = T.matrix()
y = T.tanh(T.dot(v, A))
results, updates = theano.scan(lambda i: T.grad(y[i], v), sequences = [T.arange(y.shape[0])])
compute_jac_t = theano.function([A, v], results, allow_input_downcast = True)
x = np.eye(5, dtype = theano.config.floatX)[0]
w = np.eye(5, 3, dtype = theano.config.floatX)
w[2] = np.ones((3), dtype = theano.config.floatX)
print(compute_jac_t(w, x))
print(((1 - np.tanh(x.dot(w)) ** 2) * w).T)
#####print
[[ 0.41997434 0. 0.41997434 0. 0. ]
[ 0. 1. 1. 0. 0. ]
[ 0. 0. 1. 0. 0. ]]
[[ 0.41997434 0. 0.41997434 0. 0. ]
[ 0. 1. 1. 0. 0. ]
[ 0. 0. 1. 0. 0. ]]
#####
注意:遍历的是 \(y\) 的索引,而不是 \(y\) 的元素. 因为 \(scan\) 函数为内部函数创建了一个占位符变量,这个占位符变量和替代它的其他变量不同.
\(Scan\) 例子:计算 \(scan\) 函数的循环次数
import theano
import theano.tensor as T
k = theano.shared(0)
n_sym = T.iscalar("n_sym")
results, updates = theano.scan(lambda: {k: (k + 1)}, n_steps = n_sym)
accumulator = theano.function([n_sym], [], updates = updates, allow_input_downcast = True)
print(k.get_value()) #print 0
accumulator(5)
print(k.get_value()) #print 5
\(Scan\) 例子:计算 \(tanh(v.dot(W) + b) * d\) 其中 \(d\) 服从二项分布
import theano
import theano.tensor as T
import numpy as np
X = T.matrix('X')
W = T.matrix('W')
b_sym = T.vector('b_sym')
trng = T.shared_randomstreams.RandomStreams(1234)
d = trng.binomial(size = W[1].shape)
results, updates = theano.scan(lambda v: T.tanh(T.dot(v, W) + b_sym) * d, sequences = X)
compute_with_bnoise = theano.function(inputs = [X, W, b_sym], outputs = results, updates = updates, allow_input_downcast = True)
x = np.eye(10, 2, dtype = theano.config.floatX)
w = np.ones((2, 2), dtype = theano.config.floatX)
b = np.ones((2), dtype = theano.config.floatX)
print(compute_with_bnoise(x, w, b))
#####print
[[ 0.96402758 0. ]
[ 0. 0.96402758]
[ 0. 0. ]
[ 0.76159416 0.76159416]
[ 0.76159416 0. ]
[ 0. 0.76159416]
[ 0. 0.76159416]
[ 0. 0.76159416]
[ 0. 0. ]
[ 0.76159416 0.76159416]]
#####
注意:假如你不想随机变量 \(d\) 在 \(scan\) 函数循环是发生变换,就应该把它作为 \(non_sequences\) 参数.
**\(Scan\) 例子:计算 \(pow(A, k)\) **
import theano
import theano.tensor as T
theano.config.warn.subtensor_merge_bug = False
k = T.iscalar('k')
A = T.vector('A')
def inner_fct(prior_result, B):
return prior_result * B
result, updates = theano.scan(fn = inner_fct, outputs_info = T.ones_like(A), non_sequences = A, n_steps = k)
final_result = result[-1]
power = theano.function(inputs = [A, k], outputs = final_result, updates = updates)
print(power(range(10), 2))
#####print
[ 0. 1. 4. 9. 16. 25. 36. 49. 64. 81.]
#####
**\(Scan\) 例子:计算多项式 **
import theano
import theano.tensor as T
import numpy
theano.config.warn.subtensor_merge_bug = False
coefficients = theano.tensor.vector("coefficients")
x = T.scalar('x')
max_coefficients_supported = 10000
full_range = theano.tensor.arange(max_coefficients_supported)
components, updates = theano.scan(fn = lambda coeff, power, free_var: coeff * (free_var ** power), outputs_info = None, sequences = [coefficients, full_range], non_sequences = x)
polynomial = components.sum()
calculate_polynomial = theano.function(inputs = [coefficients, x], outputs = polynomial)
test_coeff = numpy.asarray([1, 0, 2], dtype = numpy.float32)
print(calculate_polynomial(test_coeff, 3))
#####print
19.0
#####