xavier_ normal_ Initialization test

  Reference resources Deep feedforward network and Xavier Initialization principle - You know (zhihu.com)

xavier_normal_ initialization , Input in hypothesis x The mean of 0 Under the circumstances , Output can be maintained y With the input x The variance of is constant , Test with code

import torch


def get_linear(vavier=False):
    linear = torch.nn.Linear(300, 400)
    if vavier:
        torch.nn.init.xavier_normal_(linear.weight)
        torch.nn.init.kaiming_uniform_()
    return linear


def get_x():
    x = 9 * torch.randn(300, 300)
    return x


if __name__ == '__main__':
    x = get_x()
    linear = get_linear(True)
    linear2 = get_linear()
    relu = torch.nn.ReLU()
    print(f" Raw input x The mean for {x.mean()},  The variance of {x.var()}")
    # print(f"linear layer w The mean for {linear.weight.mean()},  The variance of {linear.weight.var()}")
    x_out = linear(x)
    x_out2 = linear2(x)
    print(f"x after xavier initialization linear After the layer w The mean for {x_out.mean()},  The variance of {x_out.var()}")
    print(f"x after kaiming initialization linear After the layer w The mean for {x_out2.mean()},  The variance of {x_out2.var()}")
    x_out = relu(x_out)
    x_out2 = relu(x_out2)
    print(f"x after xavier,relu After the layer w The mean for {x_out.mean()},  The variance of {x_out.var()}")
    print(f"x after kaiming,relu After the layer w The mean for {x_out2.mean()},  The variance of {x_out2.var()}")