除了庞大的数据集和强大的硬件， 优秀的软件工具在深度学习的快速发展中发挥了不可或缺的作用。 从2007年发布的开创性的Theano库开始， 灵活的开源工具使研究人员能够快速开发模型原型， 避免了我们使用标准组件时的重复工作， 同时仍然保持了我们进行底层修改的能力。 随着时间的推移，深度学习库已经演变成提供越来越粗糙的抽象。 就像半导体设计师从指定晶体管到逻辑电路再到编写代码一样， 神经网络研究人员已经从考虑单个人工神经元的行为转变为从层的角度构思网络， 通常在设计架构时考虑的是更粗糙的块（block）。
层和块
import tensorflow as tf
import numpy as np
net = tf.keras.models.Sequential([
tf.keras.layers.Dense(256, activation=tf.nn.relu),
tf.keras.layers.Dense(10)
])
X = tf.random.uniform((2, 20))
net(X)
"""
<tf.Tensor: shape=(2, 10), dtype=float32, numpy=
array([[-0.04290408, 0.16185862, 0.33581334, -0.4772908 , -0.43052262,
-0.2755071 , -0.01364495, -0.01924989, -0.25683147, 0.14087945],
[ 0.08288381, 0.25238007, 0.2683103 , -0.31998044, -0.1930172 ,
-0.47576165, -0.0759996 , 0.15379383, -0.42053652, 0.01198682]],
dtype=float32)>
"""
自定义块
class MLP(tf.keras.Model):
# 用模型参数声明层。这里，我们声明两个全连接的层
def __init__(self):
# 调用MLP的父类Model的构造函数来执行必要的初始化
# 这样，在类实例化时也可以指定其他函数参数，例如模型参数params
super().__init__()
# Hidden layer
self.hidden = tf.keras.layers.Dense(units=256, activation=tf.nn.relu)
self.out = tf.keras.layers.Dense(units=10) # Output layer
# 定义模型的前向传播，即如何根据输入X返回所需的模型输出
def call(self, X):
return self.out(self.hidden(X))
net = MLP()
net(X)
"""
<tf.Tensor: shape=(2, 10), dtype=float32, numpy=
array([[-0.26962578, -0.11410601, 0.07386565, 0.14375813, -0.108279 ,
0.26483834, -0.03660164, -0.08067364, 0.10585064, 0.24364512],
[-0.24622872, -0.12058591, 0.06265444, -0.04843516, -0.0802037 ,
-0.12859441, -0.08167031, -0.00575764, 0.0458674 , 0.09763081]],
dtype=float32)>
"""
顺序块
class MySequential(tf.keras.Model):
def __init__(self, *args):
super().__init__()
self.modules = []
for block in args:
# 这里block是tf.keras.layers.Layer子类的一个实例
self.modules.append(block)
def call(self, X):
for module in self.modules:
X = module(X)
return X
net = MySequential(
tf.keras.layers.Dense(units=256, activation=tf.nn.relu),
tf.keras.layers.Dense(10)
)
net(X)
"""
<tf.Tensor: shape=(2, 10), dtype=float32, numpy=
array([[-1.17208004e-01, -7.67507404e-03, 4.20906067e-01,
2.63195217e-01, 6.44621402e-02, -1.11673675e-01,
8.89514387e-02, -1.46050304e-01, -6.07277453e-02,
-3.42084467e-02],
[ 2.96452016e-01, 1.08508758e-01, 3.60923648e-01,
3.58375251e-01, 1.96460947e-01, -9.69487429e-02,
-3.03983688e-04, -4.94183600e-01, 5.83917871e-02,
8.36815685e-02]], dtype=float32)>
"""
在前向传播函数中执行代码
class FixedHiddenMLP(tf.keras.Model):
def __init__(self):
super().__init__()
self.flatten = tf.keras.layers.Flatten()
# 使用tf.constant函数创建的随机权重参数在训练期间不会更新（即为常量参数）
self.rand_weight = tf.constant(tf.random.uniform((20, 20)))
self.dense = tf.keras.layers.Dense(20, activation=tf.nn.relu)
def call(self, inputs):
X = self.flatten(inputs)
# 使用创建的常量参数以及relu和matmul函数
"""
我们实现了一个隐藏层， 其权重（self.rand_weight）在实例化时被随机初始化，之后为常量。
这个权重不是一个模型参数，因此它永远不会被反向传播更新。 然后，神经网络将这个固定层的输出通过一个全连接层。
"""
X = tf.nn.relu(tf.matmul(X, self.rand_weight) + 1)
# 复用全连接层。这相当于两个全连接层共享参数
X = self.dense(X)
# 控制流
while tf.reduce_sum(tf.math.abs(X)) > 1:
X /= 2
return tf.reduce_sum(X)
net = FixedHiddenMLP()
net(X)
"""
<tf.Tensor: shape=(), dtype=float32, numpy=0.58207244>
"""
class NestMLP(tf.keras.Model):
def __init__(self):
super().__init__()
self.net = tf.keras.Sequential()
self.net.add(tf.keras.layers.Dense(64, activation=tf.nn.relu))
self.net.add(tf.keras.layers.Dense(32, activation=tf.nn.relu))
self.dense = tf.keras.layers.Dense(16, activation=tf.nn.relu)
def call(self, inputs):
return self.dense(self.net(inputs))
chimera = tf.keras.Sequential()
chimera.add(NestMLP())
chimera.add(tf.keras.layers.Dense(20))
chimera.add(FixedHiddenMLP())
chimera(X)
"""
<tf.Tensor: shape=(), dtype=float32, numpy=0.8986759>
"""
参数访问
net = tf.keras.models.Sequential([
tf.keras.layers.Flatten(),
tf.keras.layers.Dense(4, activation=tf.nn.relu),
tf.keras.layers.Dense(1)
])
X = tf.random.uniform((2, 4))
net(X)
"""
<tf.Tensor: shape=(2, 1), dtype=float32, numpy=
array([[0.1116072 ],
[0.02088086]], dtype=float32)>
"""
print(net.layers[2].weights)
"""
[<tf.Variable 'dense_14/kernel:0' shape=(4, 1) dtype=float32, numpy=
array([[ 0.03134096],
[ 0.51048875],
[ 0.20824826],
[-0.5918182 ]], dtype=float32)>, <tf.Variable 'dense_14/bias:0' shape=(1,) dtype=float32, numpy=array([0.], dtype=float32)>]
"""
print(type(net.layers[2].weights[1]))
print(net.layers[2].weights[1])
print(tf.convert_to_tensor(net.layers[2].weights[1]))
"""
<class 'tensorflow.python.ops.resource_variable_ops.ResourceVariable'>
<tf.Variable 'dense_14/bias:0' shape=(1,) dtype=float32, numpy=array([0.], dtype=float32)>
tf.Tensor([0.], shape=(1,), dtype=float32)
"""
print(net.layers[1].weights)
print(net.get_weights())
"""
[<tf.Variable 'dense_13/kernel:0' shape=(4, 4) dtype=float32, numpy=
array([[-0.3207549 , 0.08903289, 0.43716985, 0.1183672 ],
[-0.35017866, -0.21974826, -0.167526 , -0.52557135],
[ 0.6917662 , 0.45998472, 0.09678668, 0.5109622 ],
[ 0.7497603 , -0.71563303, 0.4555226 , -0.4751711 ]],
dtype=float32)>, <tf.Variable 'dense_13/bias:0' shape=(4,) dtype=float32, numpy=array([0., 0., 0., 0.], dtype=float32)>]
[array([[-0.3207549 , 0.08903289, 0.43716985, 0.1183672 ],
[-0.35017866, -0.21974826, -0.167526 , -0.52557135],
[ 0.6917662 , 0.45998472, 0.09678668, 0.5109622 ],
[ 0.7497603 , -0.71563303, 0.4555226 , -0.4751711 ]],
dtype=float32), array([0., 0., 0., 0.], dtype=float32), array([[ 0.03134096],
[ 0.51048875],
[ 0.20824826],
[-0.5918182 ]], dtype=float32), array([0.], dtype=float32)]
"""
net.get_weights()[1] # 偏置项参数
"""
array([0., 0., 0., 0.], dtype=float32)
"""
def block1(name):
return tf.keras.Sequential([
# X是二维的，也就是说没有batch的维度，因此其实没有起作用，经过flatten维度不变
tf.keras.layers.Flatten(),
tf.keras.layers.Dense(4, activation=tf.nn.relu, name=name)
])
def block2():
net = tf.keras.Sequential()
for i in range(4):
# 在这里嵌套
net.add(block1(name=f'block-{i}'))
return net
rgnet = tf.keras.Sequential()
rgnet.add(block2())
rgnet.add(tf.keras.layers.Dense(1))
rgnet(X)
"""
<tf.Tensor: shape=(2, 1), dtype=float32, numpy=
array([[0.],
[0.]], dtype=float32)>
"""
print(rgnet.summary()) # 4*(4+1) + 4*(4+1) + 4*(4+1) + 4*(4+1) + (4+1)
"""
Model: "sequential_16"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
sequential_17 (Sequential) (2, 4) 80
_________________________________________________________________
dense_17 (Dense) (2, 1) 5
=================================================================
Total params: 85
Trainable params: 85
Non-trainable params: 0
_________________________________________________________________
None
"""
rgnet.layers[0].layers[1].layers[1].weights[1]
"""
<tf.Variable 'block-1/bias:0' shape=(4,) dtype=float32, numpy=array([0., 0., 0., 0.], dtype=float32)>
"""
参数初始化
# 内置初始化
net = tf.keras.models.Sequential([
tf.keras.layers.Flatten(),
tf.keras.layers.Dense(
4,
activation=tf.nn.relu,
kernel_initializer=tf.random_normal_initializer(mean=0, stddev=0.01),
bias_initializer=tf.zeros_initializer()
),
tf.keras.layers.Dense(1)
])
net(X)
"""
<tf.Tensor: shape=(2, 1), dtype=float32, numpy=
array([[ 0.0004508 ],
[-0.00056325]], dtype=float32)>
"""
net.weights[0], net.weights[1]
"""
(<tf.Variable 'dense_18/kernel:0' shape=(4, 4) dtype=float32, numpy=
array([[ 0.00936705, -0.00546127, 0.00297332, 0.00961739],
[ 0.00299437, -0.01580729, -0.00122335, 0.00139136],
[-0.00579328, 0.01332515, 0.01399793, 0.00444172],
[-0.00757714, -0.01153165, -0.00556281, -0.00651804]],
dtype=float32)>,
<tf.Variable 'dense_18/bias:0' shape=(4,) dtype=float32, numpy=array([0., 0., 0., 0.], dtype=float32)>)
"""
net = tf.keras.models.Sequential([
tf.keras.layers.Flatten(),
tf.keras.layers.Dense(
4,
activation=tf.nn.relu,
kernel_initializer=tf.keras.initializers.Constant(1),
bias_initializer=tf.zeros_initializer()
),
tf.keras.layers.Dense(1)
])
net(X)
"""
<tf.Tensor: shape=(2, 1), dtype=float32, numpy=
array([[1.6196353],
[0.4654615]], dtype=float32)>
"""
net.weights[0], net.weights[1]
"""
(<tf.Variable 'dense_20/kernel:0' shape=(4, 4) dtype=float32, numpy=
array([[1., 1., 1., 1.],
[1., 1., 1., 1.],
[1., 1., 1., 1.],
[1., 1., 1., 1.]], dtype=float32)>,
<tf.Variable 'dense_20/bias:0' shape=(4,) dtype=float32, numpy=array([0., 0., 0., 0.], dtype=float32)>)
"""
net = tf.keras.models.Sequential([
tf.keras.layers.Flatten(),
tf.keras.layers.Dense(4, activation=tf.nn.relu, kernel_initializer=tf.keras.initializers.GlorotUniform()),
tf.keras.layers.Dense(1, kernel_initializer=tf.keras.initializers.Constant(1))
])
net(X)
"""
<tf.Tensor: shape=(2, 1), dtype=float32, numpy=
array([[1.1036941 ],
[0.24121845]], dtype=float32)>
"""
print(net.layers[1].weights[0])
print(net.layers[2].weights[0])
"""
<tf.Variable 'dense_22/kernel:0' shape=(4, 4) dtype=float32, numpy=
array([[ 0.3928185 , 0.19452113, -0.6222625 , 0.5098383 ],
[ 0.7411222 , -0.6079396 , 0.7660083 , 0.3188972 ],
[-0.20573944, -0.5214578 , -0.59844893, -0.67086643],
[-0.29475045, 0.20483035, -0.06896585, 0.02373922]],
dtype=float32)>
<tf.Variable 'dense_23/kernel:0' shape=(4, 1) dtype=float32, numpy=
array([[1.],
[1.],
[1.],
[1.]], dtype=float32)>
"""
# 自定义初始化
"""
在这里，我们定义了一个Initializer的子类， 并实现了__call__函数。 该函数返回给定形状和数据类型的所需张量。
"""
class MyInit(tf.keras.initializers.Initializer):
def __call__(self, shape, dtype=None):
data = tf.random.uniform(shape, -10, 10, dtype=dtype)
factor = tf.abs(data) >= 5
factor = tf.cast(factor, tf.float32)
return data * factor
net = tf.keras.models.Sequential([
tf.keras.layers.Flatten(),
tf.keras.layers.Dense(4, activation=tf.nn.relu, kernel_initializer=MyInit()),
tf.keras.layers.Dense(1)
])
net(X)
"""
<tf.Tensor: shape=(2, 1), dtype=float32, numpy=
array([[2.4390702],
[1.3394781]], dtype=float32)>
"""
print(net.layers[1].weights[0])
"""
<tf.Variable 'dense_24/kernel:0' shape=(4, 4) dtype=float32, numpy=
array([[-0. , 9.192398 , 8.276976 , 0. ],
[-9.562297 , 0. , 6.1546516, 0. ],
[ 8.504074 , 0. , 0. , -7.715745 ],
[-8.956351 , 0. , 7.2680073, 0. ]], dtype=float32)>
"""
# 注意，我们始终可以直接设置参数
net.layers[1].weights[0][:].assign(net.layers[1].weights[0] + 1)
net.layers[1].weights[0][0, 0].assign(42)
net.layers[1].weights[0]
"""
<tf.Variable 'dense_24/kernel:0' shape=(4, 4) dtype=float32, numpy=
array([[42. , 12.192398 , 11.276976 , 3. ],
[-6.562297 , 3. , 9.154652 , 3. ],
[11.504074 , 3. , 3. , -4.715745 ],
[-5.9563513, 3. , 10.268007 , 3. ]], dtype=float32)>
"""
参数绑定
"""
有时我们希望在多个层间共享参数： 我们可以定义一个稠密层，然后使用它的参数来设置另一个层的参数。
"""
# tf.keras的表现有点不同，它会自动删除重复层
shared = tf.keras.layers.Dense(4, activation=tf.nn.relu)
net = tf.keras.models.Sequential([
tf.keras.layers.Flatten(),
shared,
shared,
tf.keras.layers.Dense(1)
])
net(X)
"""
<tf.Tensor: shape=(2, 1), dtype=float32, numpy=
array([[-0.02321801],
[ 0.01264826]], dtype=float32)>
"""
# 检查参数是否不同
print(len(net.layers) == 3) # 只剩下flatten，shared，dense三个层
"""
True
"""
延后初始化
net = tf.keras.models.Sequential([
tf.keras.layers.Dense(256, activation=tf.nn.relu),
tf.keras.layers.Dense(10)
])
"""
因为输入维数是未知的，所以网络不可能知道输入层权重的维数。 因此，框架尚未初始化任何参数，我们通过尝试访问以下参数进行确认
"""
[net.layers[i].get_weights() for i in range(len(net.layers))]
"""
[[], []]
"""
X = tf.random.uniform((2, 20))
net(X)
"""
<tf.Tensor: shape=(2, 10), dtype=float32, numpy=
array([[-0.11386063, -0.00487196, 0.04176567, 0.1878469 , 0.10872965,
-0.16000062, -0.05067926, -0.07579428, 0.08672158, -0.10469792],
[-0.15152982, -0.07497732, 0.15857704, 0.13723594, 0.32058844,
-0.15092999, 0.09441049, -0.10225398, 0.17057276, -0.02920698]],
dtype=float32)>
"""
[w.shape for w in net.get_weights()]
"""
[(20, 256), (256,), (256, 10), (10,)]
"""
不带参数的层
class CenteredLayer(tf.keras.Model):
def __init__(self):
super().__init__()
def call(self, inputs):
return inputs - tf.reduce_mean(inputs)
layer = CenteredLayer()
layer(tf.constant([1, 2, 3, 4, 5]))
"""
<tf.Tensor: shape=(5,), dtype=int32, numpy=array([-2, -1, 0, 1, 2])>
"""
net = tf.keras.Sequential([tf.keras.layers.Dense(256), CenteredLayer()])
Y = net(tf.random.uniform((4, 8)))
tf.reduce_mean(Y)
"""
<tf.Tensor: shape=(), dtype=float32, numpy=0.0>
"""
带参数的层
class MyDense(tf.keras.Model):
def __init__(self, units):
super().__init__()
self.units = units
def build(self, X_shape):
self.weight = self.add_weight(name='weight', shape=[X_shape[-1], self.units], initializer=tf.random_normal_initializer())
self.bias = self.add_weight(name='bias', shape=[self.units], initializer=tf.zeros_initializer())
def call(self, X):
linear = tf.matmul(X, self.weight) + self.bias
return tf.nn.relu(linear)
dense = MyDense(3)
dense(tf.random.uniform((2, 5)))
dense.get_weights()
"""
[array([[ 0.00817673, 0.02359941, -0.02445445],
[-0.01430137, 0.07688206, 0.00117064],
[-0.01817204, -0.00838405, 0.05307764],
[ 0.05277959, -0.00456494, 0.01542496],
[-0.01230057, 0.05387021, -0.00627117]], dtype=float32),
array([0., 0., 0.], dtype=float32)]
"""
dense(tf.random.uniform((2, 5)))
"""
<tf.Tensor: shape=(2, 3), dtype=float32, numpy=
array([[0.03652133, 0.06277972, 0.00342073],
[0.00039518, 0.0822235 , 0.02796303]], dtype=float32)>
"""
net = tf.keras.models.Sequential([MyDense(8), MyDense(1)])
net(tf.random.uniform((2, 64)))
"""
<tf.Tensor: shape=(2, 1), dtype=float32, numpy=
array([[0.01318499],
[0. ]], dtype=float32)>
"""
加载和保存张量
x = tf.range(4)
np.save('x-file.npy', x)
x2 = np.load('x-file.npy', allow_pickle=True)
x2
"""
array([0, 1, 2, 3])
"""
y = tf.zeros(4)
np.save('xy-files.npy', [x, y])
x2, y2 = np.load('xy-files.npy', allow_pickle=True)
(x2, y2)
"""
(array([0., 1., 2., 3.]), array([0., 0., 0., 0.]))
"""
mydict = {'x': x, 'y': y}
np.save('mydict.npy', mydict)
mydict2 = np.load('mydict.npy', allow_pickle=True)
mydict2
"""
array({'x': <tf.Tensor: shape=(4,), dtype=int32, numpy=array([0, 1, 2, 3])>, 'y': <tf.Tensor: shape=(4,), dtype=float32, numpy=array([0., 0., 0., 0.], dtype=float32)>},
dtype=object)
"""
class MLP(tf.keras.Model):
def __init__(self):
super().__init__()
self.flatten = tf.keras.layers.Flatten()
self.hidden = tf.keras.layers.Dense(units=256, activation=tf.nn.relu)
self.out = tf.keras.layers.Dense(units=10)
def call(self, inputs):
x = self.flatten(inputs)
x = self.hidden(x)
return self.out(x)
net = MLP()
X = tf.random.uniform((2, 20))
Y = net(X)
net.save_weights('mlp.params')
clone = MLP()
clone.load_weights('mlp.params')
"""
<tensorflow.python.training.tracking.util.CheckpointLoadStatus at 0x2688525d7c0>
"""
Y_clone = clone(X)
Y_clone == Y
"""
<tf.Tensor: shape=(2, 10), dtype=bool, numpy=
array([[ True, True, True, True, True, True, True, True, True,True], [ True, True, True, True, True, True, True, True, True, True]])>
"""
GPU
!nvidia-smi
"""
Sat Apr 22 14:38:27 2023
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 526.47 Driver Version: 526.47 CUDA Version: 12.0 |
|-------------------------------+----------------------+----------------------+
| GPU Name TCC/WDDM | Bus-Id Disp.A | Volatile Uncorr. ECC |
| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |
| | | MIG M. |
|===============================+======================+======================|
| 0 NVIDIA GeForce ... WDDM | 00000000:01:00.0 On | N/A |
| 0% 47C P8 30W / 350W | 23905MiB / 24576MiB | 1% Default |
| | | N/A |
+-------------------------------+----------------------+----------------------+
+-----------------------------------------------------------------------------+
| Processes: |
| GPU GI CI PID Type Process name GPU Memory |
| ID ID Usage |
|=============================================================================|
| 0 N/A N/A 3240 C+G ...me\Application\chrome.exe N/A |
| 0 N/A N/A 7176 C+G C:\Windows\explorer.exe N/A |
| 0 N/A N/A 7532 C+G ...7.3-Beta-x64\Snipaste.exe N/A |
| 0 N/A N/A 8276 C+G ...n1h2txyewy\SearchHost.exe N/A |
| 0 N/A N/A 8316 C+G ...cw5n1h2txyewy\LockApp.exe N/A |
| 0 N/A N/A 9148 C+G ...artMenuExperienceHost.exe N/A |
| 0 N/A N/A 11064 C+G ...2txyewy\TextInputHost.exe N/A |
| 0 N/A N/A 11788 C+G ...ge\Application\msedge.exe N/A |
| 0 N/A N/A 12380 C+G ...perience\NVIDIA Share.exe N/A |
| 0 N/A N/A 12668 C+G ...perience\NVIDIA Share.exe N/A |
| 0 N/A N/A 13468 C+G ...lPanel\SystemSettings.exe N/A |
| 0 N/A N/A 13956 C+G ...wekyb3d8bbwe\Video.UI.exe N/A |
| 0 N/A N/A 17232 C+G ...722.39\msedgewebview2.exe N/A |
| 0 N/A N/A 17620 C ...\envs\python38\python.exe N/A |
| 0 N/A N/A 20240 C+G ...e\PhoneExperienceHost.exe N/A |
| 0 N/A N/A 20356 C+G ...oft\OneDrive\OneDrive.exe N/A |
+-----------------------------------------------------------------------------+
"""
tf.device('/CPU:0'), tf.device('/GPU:0')
"""
(<tensorflow.python.eager.context._EagerDeviceContext at 0x26882b41c00>,
<tensorflow.python.eager.context._EagerDeviceContext at 0x26885287cc0>)
"""
len(tf.config.experimental.list_physical_devices('GPU'))
"""
1
"""
import tensorflow as tf
gpus = tf.config.experimental.list_physical_devices('GPU')
tf.config.experimental.set_virtual_device_configuration(
gpus[0],
[
tf.config.experimental.VirtualDeviceConfiguration(memory_limit=9182),
tf.config.experimental.VirtualDeviceConfiguration(memory_limit=9182)
]
)
len(tf.config.experimental.list_logical_devices('GPU'))
"""
2
"""
def try_gpu(i=0):
# 如果存在，则返回gpu(i)，否则返回cpu()
if len(tf.config.experimental.list_logical_devices('GPU')) >= i+1:
return tf.device(f'/GPU:{i}')
return tf.device('/CPU:0')
def try_all_gpus():
# 返回所有可用的GPU，如果没有GPU，则返回CPU
num_gpus = len(tf.config.experimental.list_logical_devices('GPU'))
devices = [tf.device(f'/GPU:{i}') for i in range(num_gpus)]
return devices if devices else [tf.device('/CPU:0')]
try_gpu(), try_gpu(10), try_all_gpus()
"""
(<tensorflow.python.eager.context._EagerDeviceContext at 0x17e9ca0d2c0>,
<tensorflow.python.eager.context._EagerDeviceContext at 0x17e86ac7a80>,
[<tensorflow.python.eager.context._EagerDeviceContext at 0x17e9ca9e9c0>,
<tensorflow.python.eager.context._EagerDeviceContext at 0x17e9cac5740>])
vice
x = tf.constant([1, 2, 3])
"""
x = tf.constant([1, 2, 3])
x.device
"""
'/job:localhost/replica:0/task:0/device:GPU:0'
"""
"""
需要注意的是，无论何时我们要对多个项进行操作， 它们都必须在同一个设备上。
例如，如果我们对两个张量求和， 我们需要确保两个张量都位于同一个设备上，
否则框架将不知道在哪里存储结果，甚至不知道在哪里执行计算。
"""
# 存储在GPU上
with try_gpu():
X = tf.ones((2, 3))
X
"""
<tf.Tensor: shape=(2, 3), dtype=float32, numpy=
array([[1., 1., 1.],
[1., 1., 1.]], dtype=float32)>
"""
with try_gpu(1):
Y = tf.random.uniform((2, 3))
Y
"""
<tf.Tensor: shape=(2, 3), dtype=float32, numpy=
array([[0.09042239, 0.88034606, 0.12038887],
[0.18857598, 0.9658278 , 0.82404184]], dtype=float32)>
"""
with try_gpu(1):
Z = X
print(X)
print(Z)
"""
tf.Tensor(
[[1. 1. 1.]
[1. 1. 1.]], shape=(2, 3), dtype=float32)
tf.Tensor(
[[1. 1. 1.]
[1. 1. 1.]], shape=(2, 3), dtype=float32)
"""
Y + Z
"""
<tf.Tensor: shape=(2, 3), dtype=float32, numpy=
array([[1.0904224, 1.8803461, 1.1203889],
[1.188576 , 1.9658278, 1.8240418]], dtype=float32)>
"""
"""
假设变量Z已经存在于第二个GPU上。 如果我们仍然在同一个设备作用域
下调用Z2 = Z会发生什么？ 它将返回Z，而不会复制并分配新内存
"""
with try_gpu(1):
Z2 = Z
Z2 is Z
"""
True
"""
# 类似地，神经网络模型可以指定设备。 下面的代码将模型参数放在GPU上。
strategy = tf.distribute.MirroredStrategy()
with strategy.scope():
net = tf.keras.models.Sequential([tf.keras.layers.Dense(1)])
"""
WARNING:tensorflow:NCCL is not supported when using virtual GPUs, fallingback to reduction to one device
INFO:tensorflow:Using MirroredStrategy with devices ('/job:localhost/replica:0/task:0/device:GPU:0', '/job:localhost/replica:0/task:0/device:GPU:1')
"""
net(X)
"""
<tf.Tensor: shape=(2, 1), dtype=float32, numpy=
array([[-1.265375],
[-1.265375]], dtype=float32)>
"""
net.layers[0].weights[0].device, net.layers[0].weights[1].device
"""
('/job:localhost/replica:0/task:0/device:GPU:0',
'/job:localhost/replica:0/task:0/device:GPU:0')
"""
