How to access the weight matrix of gluon.nn.Dense in `hybrid_forward`?


#1

I want to do some calculations on the weights of some gluon layers (e.g., the W of nn.Dense, the conv filters of nn.Conv2D). What’s the proper way to do that? Should I define my own layers instead of using the gluon APIs? In the following example, I got errors like:

AssertionError: Argument data must be Symbol instances, but got Parameter mynet2_dense0_weight (shape=(512, 1024), dtype=float32)

import mxnet as mx
from mxnet.gluon import nn

class MyNet(nn.HybridBlock):
    def __init__(self, **kwargs):
        super(MyNet, self).__init__(**kwargs)
        with self.name_scope():
            self.layer1 = nn.Dense(512, in_units=1024, activation=None, use_bias=False)

    def hybrid_forward(self, F, x):
        x = self.layer1(x)
        tmp = F.dot(self.layer1.weight, F.transpose(self.layer1.weight))
        return x, tmp

net = MyNet()
net.initialize()
net.hybridize()

data = mx.random.randn(10, 1024)
x,tmp = net(data)

#2

you need to call .data() to get the value of your tensors as in:
self.layer1.weight.data()


#3

weight.data() returns the NDArray object which cause another error:

AssertionError: Argument data must be Symbol instances, but got 
[[ 0.0068339   0.01299825  0.0301265  ... -0.04791416  0.04603841
  -0.040671  ]
 [-0.00896072 -0.04408851 -0.06637033 ...  0.0692328   0.04998931
  -0.02321438]
 [ 0.06538417  0.05608701  0.00661252 ...  0.03893644  0.05445
  -0.00188201]
 ...
 [-0.05866455 -0.06207159  0.06228637 ...  0.06298847 -0.05717352
   0.06576373]
 [ 0.03625342  0.01019854 -0.04665761 ... -0.00459134 -0.06867207
  -0.02224994]
 [-0.05084752  0.04445304  0.06729832 ... -0.06452031  0.02575064
  -0.04114719]]
<NDArray 512x1024 @cpu(0)>

#4

Indeed @jonbakerfish, my bad, if you want the symbol, use .var()

import mxnet as mx
from mxnet.gluon import nn

class MyNet(nn.HybridBlock):
    def __init__(self, **kwargs):
        super(MyNet, self).__init__(**kwargs)
        with self.name_scope():
            self.layer1 = nn.Dense(512, in_units=1024, activation=None, use_bias=False)

    def hybrid_forward(self, F, x):
        x = self.layer1(x)
        tmp = F.dot(self.layer1.weight.var(), F.transpose(self.layer1.weight.var()))
        return x, tmp

net = MyNet()
net.initialize()
net.hybridize()

data = mx.random.randn(10, 1024)
x,tmp = net(data)