Neural Network julia Package

2023-06-08 · Prakash

In a seris of previous posts we, described how to make a fully functional Neural Network in julia. Lets create a package so that we can import the neural network in julia as a regular package available in every session.

Lets call this new package PNN. As described in this post create a directory PNN somewhere in a path.

$ mkdir -p /some/path/PNN/src
$ cd /some/path
/some/path $ ls
src

Inside the src directory create a file utils.jl with the following content

export S, S′, diffsq

S(x) = 1 ./ (1 .+ exp.(-x))
S′(x) = exp.(-x) ./ ( 1 .+ exp.(-x)).^2
diffsq(y,ŷ) = sum((y - ŷ) .^2)

Here we have defined three different functions. The activation function S, its derivatibve S' and a handy function to calculate the sum of square of differences.

\[ \begin{aligned}\sum_i (y_i - \hat y_i)^2\end{aligned} \]

As described in this post, this is just a sigmoid activation function.

Now in a file called dnn.jl write the following.

using Distributions
export NN
export feed!, fit!, train!,loss
struct NN
    W::Vector{Matrix{Float64}}
    b::Vector{Vector{Float64}}
    z::Vector{Vector{Float64}} # for speeding things up
    a::Vector{Vector{Float64}} # for speeding things up
    function NN(idim::Int64,odim::Int64;hl=[2])
        Wt,bt = [],[]
        p = idim 
        for l in vcat(hl,odim)
            d = Normal(0,1/l)
            Wl = rand(d,p,l) .* 0.1
            bl = vec(rand(Normal(0,1),l)) .* 0.1
            push!(Wt,Wl)
            push!(bt,bl)
            p = l
        end
        zt = Vector{Vector{Float64}}(undef,length(bt))
        at = Vector{Vector{Float64}}(undef,length(bt)+1)
        new(Wt,bt,zt,at)
      end
     function NN(Ws::Vector,Bs::Vector)
        zs = Vector{Vector{Float64}}(undef,length(bs))
        as = Vector{Vector{Float64}}(undef,length(bs)+1)
        new(Ws,bs,zs,as)
    end
end 

Base.show(io::IO, t::NN) = print(io,"NN $(length(t.W)-1) hidden layers" )


function feed!(nn::NN,x::Vector{Float64},σ=S)
    #print(x,W,b)
    nn.a[1] = x
    for ly in 1:length(nn.W)
        nn.z[ly] = nn.W[ly]' * nn.a[ly] + nn.b[ly]
        nn.a[ly+1] = σ(nn.z[ly])
    end
    return nn.a[end]
end


function backprop!(nn::NN,x,y,β,σ′=S′) 
    ∇L = (nn.a[end] - y)
    δp = ∇L 
    for lr = length(nn.W):-1:1
        δc = δp .* σ′(nn.z[lr])
        nn.W[lr] -= β .*  nn.a[lr] * δc'
        nn.b[lr] -= β .* δc
        δp =  nn.W[lr] * δc
    end
end

function train!(nn::NN,x,y,β) 
    feed!(nn,x)  
    # Calculate error
    backprop!(nn,x,y,β)
    ŷ = feed!(nn,x)
    return ŷ
end


function fit!(tnn,X,y;β=9e-3,epoch=50)
    errs = []
    for n in 1:epoch
        for (i,(x,y)) ∈ enumerate(zip(X,y))
            ŷ = train!(tnn,x,y,β)
            cerror = diffsq(y,ŷ)
            push!(errs,cerror)
        end
    end
    return errs
end

In the first part. we spend some time to create a mutable struct to carry around the variables in a single object. But otherwise all the code here is as described in this post.

NOw in a file called PNN.jl inside /some/path/src directory write the following.

module PNN
include("utils.jl")
include("dnn.jl")
end

That's it. That is all. Nada mas.

If as described in this post you also have LOAD_PATH configured then you can simply do this.

using PNN

input_nodes = 10
output_nodes = 3
hidden_layers = [8,5]

tnn = NN(input_nodes, output_nodes; hl=hidden_layers)

This is a fully functional neural network which we can train/test with some real world data. In the next post we will describe the use of MNIST handwriting dataset and use this package to train handwriting recognition. Its amazing how such a small looking code can to some amazing thing. Until then! Ciao. Halta pronto!!