Add PReLU layer

fohx13 · pavanky · commit 49bfb8e7ed8d · 2017-07-10T15:19:53.000-07:00
diff --git a/include/af/nn/Modules/Activations.hpp b/include/af/nn/Modules/Activations.hpp
@@ -48,5 +48,15 @@ namespace af
          
             autograd::Variable forward(const autograd::Variable &input);
         };
+
+        class PReLU : public Module
+        {
+        public:
+            PReLU(int size, double spread = 1.0);
+            PReLU(const autograd::Variable &w);
+         
+            autograd::Variable forward(const autograd::Variable &input);
+        };
+
     }
 }
diff --git a/src/nn/Modules/Activations.cpp b/src/nn/Modules/Activations.cpp
@@ -9,7 +9,7 @@
 
 #include <af/autograd/Functions.hpp>
 #include <af/nn/Modules/Activations.hpp>
-
+#include <af/nn/Types.hpp>
 namespace af
 {
     namespace nn
@@ -46,5 +46,26 @@ namespace af
         {
             return max(input, m_slope * input);
         }
+
+        PReLU::PReLU(int size, double spread)
+        {
+            auto w = nn::weight(size, 1, spread);
+            setParams({w});
+        }
+
+        PReLU::PReLU(const Variable &w) :
+            Module({w})
+        {
+        }
+
+        Variable PReLU::forward(const Variable &input)
+        {
+            auto tmp = max(input, 0.0);
+            auto res = expandAs(m_parameters[0],tmp) * tmp;
+            //TODO: Determine if doing the max after the mul is preferable
+            return res;
+
+        }
+
     }
 }
