1-3. Artificial Nuerons 구현 (2)

 

 

 

 

Activation Layer¶

 

In [5]:

import tensorflow as tf

from tensorflow.math import exp, maximum
from tensorflow.keras.layers import Activation

x = tf.random.normal(shape=(1, 5))

# import Activation func
sigmoid = Activation('sigmoid')
tanh = Activation('tanh')
relu = Activation('relu')

#foward propagation(Tensorflow)
y_sigmoid_tf = sigmoid(x)
y_tanh_tf = tanh(x)
y_relu_tf = relu(x)
print("x = {}\n{}".format(x.shape, x.numpy()))
print('--------------------------------------------------------------------------------------------------------')
print("sigmoid(Tensorflow) :{}\n{}".format(y_sigmoid_tf.shape, y_sigmoid_tf.numpy()))
print("Tanh(Tensorflow) :{}\n{}".format(y_tanh_tf.shape, y_tanh_tf.numpy()))
print("ReLU(Tensorflow) :{}\n{}".format(y_relu_tf.shape, y_relu_tf.numpy()))
print('--------------------------------------------------------------------------------------------------------')

#foward propagation(Manual)
y_sigmoid_man = 1 / (1 + exp(-x))
y_tanh_man = (exp(x) - exp(-x) / exp(x) + exp(-x))
y_relu_man = maximum(x, 0)

print("sigmoid(manual) :{}\n{}".format(y_sigmoid_man.shape, y_sigmoid_man.numpy()))
print("Tanh(manual) :{}\n{}".format(y_tanh_man.shape, y_tanh_man.numpy()))
print("ReLU(manual) :{}\n{}".format(y_relu_man.shape, y_relu_man.numpy()))

 

 

x = (1, 5)
[[-0.8214905  -0.47018248 -0.9033882  -0.59634084  0.34681112]]
--------------------------------------------------------------------------------------------------------
sigmoid(Tensorflow) :(1, 5)
[[0.30544734 0.38457307 0.28835472 0.3551813  0.5858441 ]]
Tanh(Tensorflow) :(1, 5)
[[-0.6758804  -0.4383467  -0.71794367 -0.53444076  0.33354443]]
ReLU(Tensorflow) :(1, 5)
[[0.         0.         0.         0.         0.34681112]]
--------------------------------------------------------------------------------------------------------
sigmoid(manual) :(1, 5)
[[0.30544734 0.38457307 0.28835472 0.3551813  0.58584404]]
Tanh(manual) :(1, 5)
[[-2.4568982 -0.3357414 -3.217636  -0.929621   1.6217258]]
ReLU(manual) :(1, 5)
[[0.         0.         0.         0.         0.34681112]]

 

 

Tensor와 manual로 한결과가 같음을 증명함

 

 

Activation in Dense Layer¶

 

In [9]:

import tensorflow as tf

# from tensorflow.math import exp, maximum
from tensorflow.keras.layers import Dense

# inmp. setting
x = tf.random.normal(shape=(1, 5))

# imp. artificial neurons
dense_sigmoid = Dense(units = 1 , activation = 'sigmoid')
dense_tanh = Dense(units = 1 , activation = 'tanh')
dense_relu = Dense(units = 1 , activation = 'relu')

# forward propagation
y_sigmoid = dense_sigmoid(x)
y_tanh = dense_tanh(x)
y_relu = dense_relu(x)


print("AN with Sigmoid: {}\n{}".format(y_sigmoid.shape, y_sigmoid.numpy()))
print("AN with Tanh: {}\n{}".format(y_tanh.shape, y_tanh.numpy()))
print("AN with ReLU: {}\n{}".format(y_relu.shape, y_relu.numpy()))

# forward propagation(manual)
print('\n-----------------------------------------------\n')
print('Sigmoid Manual \n')
W,B = dense_sigmoid.get_weights()
z = tf.linalg.matmul(x, W) + B
a = 1 / (1 + exp(-z))

print("Activation value(Tensorflow)= {}\n{}".format(y_sigmoid.shape, y_sigmoid.numpy()))
print("Activation value(manual)= {}\n{}".format(a.shape, a.numpy()))

 

 

AN with Sigmoid: (1, 1)
[[0.24354003]]
AN with Tanh: (1, 1)
[[-0.58961546]]
AN with ReLU: (1, 1)
[[0.16749245]]

-----------------------------------------------

Sigmoid Manual 

Activation value(Tensorflow)= (1, 1)
[[0.24354003]]
Activation value(manual)= (1, 1)
[[0.24354003]]

 

 

Artificial Nuerons¶

 

In [13]:

import tensorflow as tf
from tensorflow.keras.layers import Dense
from tensorflow.math import exp, maximum

# activation = 'sigmoid'
activation = 'tanh'
# activation = 'relu'



x = tf.random.uniform(shape=(1,10)) # input setting

dense = Dense(units=1, activation=activation)

y_tf = dense(x)
W, B = dense.get_weights()
print(x.shape)
print(W.shape)
print(B.shape)


# calcuation of activation val manually
y_man = tf.linalg.matmul(x, W) +B

if activation == 'sigmoid':
  y_man = 1/(1 + exp(-y_man))
elif activation == 'tanh':
  y_man = (exp(y_man) - exp(-y_man)) / (exp(y_man) + exp(-y_man))
elif activation == 'relu':
  y_man = maximum(y_man, 0)

print("Activation : ",activation)
print("Y_tf : {}\n{}\n".format(y_tf.shape, y_tf.numpy()))
print("Y_man : {}\n{}\n".format(y_man.shape, y_man.numpy())) 

 

 

(1, 10)
(10, 1)
(1,)
Activation :  tanh
Y_tf : (1, 1)
[[-0.36786476]]

Y_man : (1, 1)
[[-0.36786476]]

 

 

Shapes of Dense Layers¶

 

In [17]:

import tensorflow as tf

from tensorflow.keras.layers import Dense

N, n_feature = 16, 10  # 16이 100000000000이 되어도 W랑 B에는 영향 없음
x = tf.random.normal(shape=(N , n_feature)) # generate minibatch

dense = Dense(units=1, activation='relu') # input Arti Nue
y = dense(x) # foward propagation

W,B = dense.get_weights() # get W and B

print("shape of x : ", x.shape)
print("shape of W : ", W.shape)
print("shape of B : ", B.shape)

 

 

shape of x :  (16, 10)
shape of W :  (10, 1)
shape of B :  (1,)

 

 

Output Calculations¶

 

In [15]:

import tensorflow as tf

from tensorflow.keras.layers import Dense

N, n_feature = 8, 10
x = tf.random.normal(shape=(N, n_feature))

dense = Dense(units=1, activation='sigmoid' )
y_tf  = dense(x)

W,B = dense.get_weights()

y_man = tf.linalg.matmul(x, W) +B
y_man = 1/ (1 + tf.math.exp(-y_man))

print("Output(Tensorflow) : \n", y_tf.numpy())
print('\n-----------------------------------------------------------------------\n')
print("Output(Manual) : \n", y_man.numpy())

 

 

Output(Tensorflow) : 
 [[0.9200557 ]
 [0.8243117 ]
 [0.9015601 ]
 [0.14071383]
 [0.42172617]
 [0.16160782]
 [0.5007893 ]
 [0.2473704 ]]

-----------------------------------------------------------------------

Output(Manual) : 
 [[0.9200557 ]
 [0.8243117 ]
 [0.9015602 ]
 [0.14071381]
 [0.4217262 ]
 [0.1616078 ]
 [0.5007893 ]
 [0.24737042]]

 

In [ ]: