Manually assign weights using PyTorch
Question:
I am using Python 3.8 and PyTorch 1.7 to manually assign and change the weights and biases for a neural network. As an example, I have defined a LeNet-300-100 fully-connected neural network to train on MNIST dataset. The code for class definition is:
class LeNet300(nn.Module):
def __init__(self):
super(LeNet300, self).__init__()
# Define layers-
self.fc1 = nn.Linear(in_features = input_size, out_features = 300)
self.fc2 = nn.Linear(in_features = 300, out_features = 100)
self.output = nn.Linear(in_features = 100, out_features = 10)
self.weights_initialization()
def forward(self, x):
out = F.relu(self.fc1(x))
out = F.relu(self.fc2(out))
return self.output(out)
def weights_initialization(self):
'''
When we define all the modules such as the layers in '__init__()'
method above, these are all stored in 'self.modules()'.
We go through each module one by one. This is the entire network,
basically.
'''
for m in self.modules():
if isinstance(m, nn.Linear):
nn.init.xavier_normal_(m.weight)
nn.init.constant_(m.bias, 0)
To experiment with trying to change the weights for this model-
# Instantiate model-
mask_model = LeNet300()
To assign all of the weights in each of the layers to one (1), I use the code-
with torch.no_grad():
for layer in mask_model.state_dict():
mask_model.state_dict()[layer] = nn.parameter.Parameter(torch.ones_like(mask_model.state_dict()[layer]))
# Sanity check-
mask_model.state_dict()['fc1.weight']
This output shows that the weights are not equal to 1.
I also tried the code-
for param in mask_model.parameters():
# print(param.shape)
param = nn.parameter.Parameter(torch.ones_like(param))
But this does not work as well.
Help?
Answers:
I did this in very simple way ( just used fill_()
)
Here is the code :
import torch
import torch.nn as nn
class LeNet300(nn.Module):
def __init__(self):
super(LeNet300, self).__init__()
# Define layers-
self.fc1 = nn.Linear(in_features = 28, out_features = 300)
self.fc2 = nn.Linear(in_features = 300, out_features = 100)
self.output = nn.Linear(in_features = 100, out_features = 10)
self.weights_initialization()
def forward(self, x):
out = F.relu(self.fc1(x))
out = F.relu(self.fc2(out))
return self.output(out)
def weights_initialization(self):
'''
When we define all the modules such as the layers in '__init__()'
method above, these are all stored in 'self.modules()'.
We go through each module one by one. This is the entire network,
basically.
'''
for m in self.modules():
if isinstance(m, nn.Linear):
nn.init.xavier_normal_(m.weight)
nn.init.constant_(m.bias, 0)
mask_model = LeNet300()
with torch.no_grad():
for layer in mask_model.state_dict():
print(layer)
#print(torch.ones_like(mask_model.state_dict()[layer].data))
mask_model.state_dict()[layer].data.fill_(1)
mask_model.state_dict()['fc1.weight']
# tensor([[1., 1., 1., ..., 1., 1., 1.],
# [1., 1., 1., ..., 1., 1., 1.],
# [1., 1., 1., ..., 1., 1., 1.],
# ...,
# [1., 1., 1., ..., 1., 1., 1.],
# [1., 1., 1., ..., 1., 1., 1.],
# [1., 1., 1., ..., 1., 1., 1.]])
for param in mask_model.parameters():
param.data = nn.parameter.Parameter(torch.ones_like(param))
if we need to assign
a numpy array
to the layer weights
, we can do the following:
numpy_data= np.random.randn(6, 1, 3, 3)
conv = nn.Conv2d(1, 6, 3, 1, 1, bias=False)
with torch.no_grad():
conv.weight = nn.Parameter(torch.from_numpy(numpy_data).float())
# or
conv.weight.copy_(torch.from_numpy(numpy_data).float())
Source: https://discuss.pytorch.org/t/how-do-i-pass-numpy-array-to-conv2d-weight-for-initialization/56595/3
I am using Python 3.8 and PyTorch 1.7 to manually assign and change the weights and biases for a neural network. As an example, I have defined a LeNet-300-100 fully-connected neural network to train on MNIST dataset. The code for class definition is:
class LeNet300(nn.Module):
def __init__(self):
super(LeNet300, self).__init__()
# Define layers-
self.fc1 = nn.Linear(in_features = input_size, out_features = 300)
self.fc2 = nn.Linear(in_features = 300, out_features = 100)
self.output = nn.Linear(in_features = 100, out_features = 10)
self.weights_initialization()
def forward(self, x):
out = F.relu(self.fc1(x))
out = F.relu(self.fc2(out))
return self.output(out)
def weights_initialization(self):
'''
When we define all the modules such as the layers in '__init__()'
method above, these are all stored in 'self.modules()'.
We go through each module one by one. This is the entire network,
basically.
'''
for m in self.modules():
if isinstance(m, nn.Linear):
nn.init.xavier_normal_(m.weight)
nn.init.constant_(m.bias, 0)
To experiment with trying to change the weights for this model-
# Instantiate model-
mask_model = LeNet300()
To assign all of the weights in each of the layers to one (1), I use the code-
with torch.no_grad():
for layer in mask_model.state_dict():
mask_model.state_dict()[layer] = nn.parameter.Parameter(torch.ones_like(mask_model.state_dict()[layer]))
# Sanity check-
mask_model.state_dict()['fc1.weight']
This output shows that the weights are not equal to 1.
I also tried the code-
for param in mask_model.parameters():
# print(param.shape)
param = nn.parameter.Parameter(torch.ones_like(param))
But this does not work as well.
Help?
I did this in very simple way ( just used fill_()
)
Here is the code :
import torch
import torch.nn as nn
class LeNet300(nn.Module):
def __init__(self):
super(LeNet300, self).__init__()
# Define layers-
self.fc1 = nn.Linear(in_features = 28, out_features = 300)
self.fc2 = nn.Linear(in_features = 300, out_features = 100)
self.output = nn.Linear(in_features = 100, out_features = 10)
self.weights_initialization()
def forward(self, x):
out = F.relu(self.fc1(x))
out = F.relu(self.fc2(out))
return self.output(out)
def weights_initialization(self):
'''
When we define all the modules such as the layers in '__init__()'
method above, these are all stored in 'self.modules()'.
We go through each module one by one. This is the entire network,
basically.
'''
for m in self.modules():
if isinstance(m, nn.Linear):
nn.init.xavier_normal_(m.weight)
nn.init.constant_(m.bias, 0)
mask_model = LeNet300()
with torch.no_grad():
for layer in mask_model.state_dict():
print(layer)
#print(torch.ones_like(mask_model.state_dict()[layer].data))
mask_model.state_dict()[layer].data.fill_(1)
mask_model.state_dict()['fc1.weight']
# tensor([[1., 1., 1., ..., 1., 1., 1.],
# [1., 1., 1., ..., 1., 1., 1.],
# [1., 1., 1., ..., 1., 1., 1.],
# ...,
# [1., 1., 1., ..., 1., 1., 1.],
# [1., 1., 1., ..., 1., 1., 1.],
# [1., 1., 1., ..., 1., 1., 1.]])
for param in mask_model.parameters():
param.data = nn.parameter.Parameter(torch.ones_like(param))
if we need to assign
a numpy array
to the layer weights
, we can do the following:
numpy_data= np.random.randn(6, 1, 3, 3)
conv = nn.Conv2d(1, 6, 3, 1, 1, bias=False)
with torch.no_grad():
conv.weight = nn.Parameter(torch.from_numpy(numpy_data).float())
# or
conv.weight.copy_(torch.from_numpy(numpy_data).float())
Source: https://discuss.pytorch.org/t/how-do-i-pass-numpy-array-to-conv2d-weight-for-initialization/56595/3