Adding negate, reciprocal, subtract and divide

pavanky · umar456 · commit 8f4cfc129fef · 2017-07-06T11:55:53.000-04:00
diff --git a/examples/autograd.cpp b/examples/autograd.cpp
@@ -9,57 +9,87 @@
 
 #include <af/autograd.h>
 
+#define VERIFY(VAL) do {                                    \
+        auto res = af::allTrue<bool>(af::abs(VAL) < 1E-5);  \
+        printf("%s:%d %s\n", __FUNCTION__, __LINE__,        \
+               res ? "PASS" : "FAIL");                      \
+    } while(0)
+
 using af::autograd::Variable;
-void test1()
+void test_multiply()
 {
     auto x = Variable(af::randu(5), true);
-    af_print(x.array());
     auto y = x * x;
-    af_print(y.array());
     auto dy = Variable(af::constant(1.0, 5), false);
     y.backward(dy);
     auto dx = x.grad();
-    af_print(dx.array() - 2 * x.array());
+    VERIFY(dx.array() - 2 * x.array());
 }
 
-void test2()
+void test_multipl_add()
 {
     auto x = Variable(af::randu(5), true);
-    af_print(x.array());
     auto y = Variable(af::randu(5), true);
-    af_print(y.array());
     auto z = x * x + x * y + y * y;
     auto dz = Variable(af::constant(1.0, 5), false);
     z.backward(dz);
     auto dx = x.grad();
     auto dy = y.grad();
-    af_print(dx.array() - 2 * x.array() - y.array());
-    af_print(dy.array() - 2 * y.array() - x.array());
+    VERIFY(dx.array() - 2 * x.array() - y.array());
+    VERIFY(dy.array() - 2 * y.array() - x.array());
 }
 
-void test3()
+void test_no_calc_grad()
 {
     auto x = Variable(af::randu(5), false);
-    af_print(x.array());
     auto y = Variable(af::randu(5), true);
-    af_print(y.array());
     auto z = x * x + x * y + y * y;
     auto dz = Variable(af::constant(1.0, 5), false);
     z.backward(dz);
     auto dy = y.grad();
-    af_print(dy.array() - 2 * y.array() - x.array());
+    VERIFY(dy.array() - 2 * y.array() - x.array());
     try {
         auto dx = x.grad();
     } catch(af::exception &ex) {
         std::cout << ex.what() << std::endl;
+        return;
     }
+    printf("%s:%d No Gradient check Failed\n");
+}
+
+void test_multiply_sub()
+{
+    auto x = Variable(af::randu(5), true);
+    auto y = Variable(af::randu(5), true);
+    auto z = x * x - x * y;
+    auto dz = Variable(af::constant(1.0, 5), false);
+    z.backward(dz);
+    auto dx = x.grad();
+    auto dy = y.grad();
+    VERIFY(dx.array() - (2 * x.array() - y.array()));
+    VERIFY(dy.array() - (-x.array()));
+}
+
+void test_divide_add()
+{
+    auto x = Variable(af::randu(5), true);
+    auto y = Variable(af::randu(5), true);
+    auto z = x + x / y + y;
+    auto dz = Variable(af::constant(1.0, 5), false);
+    z.backward(dz);
+    auto dx = x.grad();
+    auto dy = y.grad();
+    VERIFY(dx.array() - (1.0 + 1.0 / y.array()));
+    VERIFY(dy.array() - (1.0 - x.array() / (y.array() * y.array())));
 }
 
 int main()
 {
     af::info();
-    test1();
-    test2();
-    test3();
+    test_multiply();
+    test_multipl_add();
+    test_no_calc_grad();
+    test_multiply_sub();
+    test_divide_add();
     return 0;
 }
diff --git a/include/af/autograd/Functions.hpp b/include/af/autograd/Functions.hpp
@@ -13,7 +13,12 @@ namespace af {
 
         class Variable;
 
-        Variable operator +(const Variable lhs, const Variable rhs);
-        Variable operator *(const Variable lhs, const Variable rhs);
+        Variable operator +(const Variable &lhs, const Variable &rhs);
+        Variable operator *(const Variable &lhs, const Variable &rhs);
+        Variable operator -(const Variable &lhs, const Variable &rhs);
+        Variable operator /(const Variable &lhs, const Variable &rhs);
+
+        Variable negate(const Variable &input);
+        Variable reciprocal(const Variable &input);
     }
 }
diff --git a/include/af/autograd/Variable.hpp b/include/af/autograd/Variable.hpp
@@ -22,16 +22,16 @@ namespace af {
         class Variable
         {
         public:
-            typedef std::function<void(std::vector<Variable>, Variable)> GradFunc_t;
+            typedef std::function<void(std::vector<Variable> &, const Variable &)> GradFunc_t;
             typedef std::unordered_map<std::ptrdiff_t, bool> Cache_t;
             typedef std::vector<Variable> DAG_t;
 
         private:
             struct Shared {
                 Shared();
-                Shared(af::array data, bool calc_grad);
-                Shared(af::array data,
-                       std::vector<Variable> inputs,
+                Shared(const af::array &data, bool calc_grad);
+                Shared(const af::array &data,
+                       const std::vector<Variable> &inputs,
                        GradFunc_t grad_func,
                        bool calc_grad);
 
@@ -45,26 +45,26 @@ namespace af {
             public:
 
             Variable();
-            Variable(af::array data, bool calc_grad);
-            Variable(af::array data,
-                     std::vector<Variable> inputs,
+            Variable(const af::array &data, bool calc_grad);
+            Variable(const af::array &data,
+                     const std::vector<Variable> &inputs,
                      GradFunc_t grad_func);
 
             af::array array() const;
 
             Variable grad() const;
 
-            bool isCalcGrad();
+            bool isCalcGrad() const;
 
             void setCalcGrad(bool calc_grad);
 
-            void addGrad(Variable child_grad);
+            void addGrad(const Variable &child_grad);
 
             void evalGrad();
 
             void calcGradInputs();
 
-            void backward(Variable grad);
+            void backward(const Variable &grad);
 
             DAG_t build();
 
diff --git a/src/autograd/Functions.cpp b/src/autograd/Functions.cpp
@@ -13,25 +13,66 @@
 namespace af {
     namespace autograd {
 
-        Variable operator +(const Variable lhs, const Variable rhs)
+        Variable operator +(const Variable &lhs, const Variable &rhs)
         {
             auto result = lhs.array() + rhs.array();
-            auto grad_func = [](std::vector<Variable> inputs, Variable grad_output) {
+            auto grad_func = [](std::vector<Variable> &inputs, const Variable &grad_output) {
                 inputs[0].addGrad(grad_output);
                 inputs[1].addGrad(grad_output);
             };
             return Variable(result, {lhs, rhs}, grad_func);
         }
 
-        Variable operator *(const Variable lhs, const Variable rhs)
+        Variable operator -(const Variable &lhs, const Variable &rhs)
+        {
+            auto result = lhs.array() - rhs.array();
+            auto grad_func = [](std::vector<Variable> &inputs, const Variable &grad_output) {
+                inputs[0].addGrad(grad_output);
+                inputs[1].addGrad(negate(grad_output));
+            };
+            return Variable(result, {lhs, rhs}, grad_func);
+        }
+
+        Variable operator *(const Variable &lhs, const Variable &rhs)
         {
             auto result = lhs.array() * rhs.array();
-            auto grad_func = [](std::vector<Variable> inputs, Variable grad_output) {
+            auto grad_func = [](std::vector<Variable> &inputs, const Variable &grad_output) {
                 inputs[0].addGrad(grad_output * inputs[1]);
                 inputs[1].addGrad(grad_output * inputs[0]);
             };
             return Variable(result, {lhs, rhs}, grad_func);
         }
 
+        Variable operator /(const Variable &lhs, const Variable &rhs)
+        {
+            auto result = lhs.array() / rhs.array();
+            auto grad_func = [](std::vector<Variable> &inputs, const Variable &grad_output) {
+                auto inputs_1_rec = reciprocal(inputs[1]);
+                auto grad_input_0 = grad_output * inputs_1_rec;
+                inputs[0].addGrad(grad_input_0);
+                inputs[1].addGrad(grad_input_0 * negate(inputs[0]) * inputs_1_rec);
+            };
+            return Variable(result, {lhs, rhs}, grad_func);
+        }
+
+        Variable negate(const Variable &input)
+        {
+            auto result = 0.0 - input.array();
+            auto grad_func = [](std::vector<Variable> &inputs, const Variable &grad_output) {
+                inputs[0].addGrad(negate(grad_output));
+            };
+            return Variable(result, {input}, grad_func);
+        }
+
+        Variable reciprocal(const Variable &input)
+        {
+            auto result = 1.0 / input.array();
+            auto grad_func = [](std::vector<Variable> &inputs, const Variable &grad_output) {
+                auto res = reciprocal(inputs[0]);
+                inputs[0].addGrad(negate(grad_output) * res * res);
+            };
+            return Variable(result, {input}, grad_func);
+        }
+
     }
 }
diff --git a/src/autograd/Variable.cpp b/src/autograd/Variable.cpp
@@ -21,16 +21,16 @@ namespace af {
             m_grad_func(nullptr)
         {}
 
-        Variable::Shared::Shared(af::array data, bool calc_grad) :
+        Variable::Shared::Shared(const af::array &data, bool calc_grad) :
             m_calc_grad(calc_grad),
             m_data(data),
             m_inputs(),
             m_grads(),
             m_grad_func(nullptr)
         {}
 
-        Variable::Shared::Shared(af::array data,
-                                 std::vector<Variable> inputs,
+        Variable::Shared::Shared(const af::array &data,
+                                 const std::vector<Variable> &inputs,
                                  GradFunc_t grad_func,
                                  bool calc_grad) :
             m_calc_grad(calc_grad),
@@ -45,13 +45,13 @@ namespace af {
         {
         }
 
-        Variable::Variable(af::array data, bool calc_grad) :
+        Variable::Variable(const af::array &data, bool calc_grad) :
             m_shared(new Shared(data, calc_grad))
         {}
 
-        Variable::Variable(af::array data,
-                 std::vector<Variable> inputs,
-                 GradFunc_t grad_func) :
+        Variable::Variable(const af::array &data,
+                           const std::vector<Variable> &inputs,
+                           GradFunc_t grad_func) :
             m_shared(nullptr)
         {
             bool calc_grad = false;
@@ -81,7 +81,7 @@ namespace af {
             return m_shared->m_grads[0];
         }
 
-        bool Variable::isCalcGrad()
+        bool Variable::isCalcGrad() const
         {
             return m_shared->m_calc_grad;
         }
@@ -96,7 +96,7 @@ namespace af {
             }
         }
 
-        void Variable::addGrad(Variable child_grad)
+        void Variable::addGrad(const Variable &child_grad)
         {
             if (m_shared->m_calc_grad) {
                 m_shared->m_grads.push_back(child_grad);
@@ -107,13 +107,17 @@ namespace af {
         {
             // Flag asking not to calculate gradients
             if (!m_shared->m_calc_grad) return;
-            Variable grad = m_shared->m_grads[0];
-            for (unsigned i = 1; i < m_shared->m_grads.size(); i++) {
-                grad = grad + m_shared->m_grads[i];
+
+            // Best not to evaluate the JIT immediately if theres only a single gradient
+            if (m_shared->m_grads.size() > 1) {
+                Variable grad = m_shared->m_grads[0];
+                for (unsigned i = 1; i < m_shared->m_grads.size(); i++) {
+                    grad = grad + m_shared->m_grads[i];
+                }
+                grad.array().eval();
+                m_shared->m_grads.clear();
+                m_shared->m_grads.push_back(grad);
             }
-            grad.array().eval();
-            m_shared->m_grads.clear();
-            m_shared->m_grads.push_back(grad);
         }
 
         void Variable::calcGradInputs()
@@ -124,7 +128,7 @@ namespace af {
             }
         }
 
-        void Variable::backward(Variable grad)
+        void Variable::backward(const Variable &grad)
         {
             this->addGrad(grad);
             Variable::DAG_t dag = this->build();

Original file line number	Diff line number	Diff line change
`@@ -13,7 +13,12 @@ namespace af {`
`13`	`13`
`14`	`14`	`class Variable;`
`15`	`15`
`16`		`- Variable operator +(const Variable lhs, const Variable rhs);`
`17`		`- Variable operator *(const Variable lhs, const Variable rhs);`
	`16`	`+ Variable operator +(const Variable &lhs, const Variable &rhs);`
	`17`	`+ Variable operator *(const Variable &lhs, const Variable &rhs);`
	`18`	`+ Variable operator -(const Variable &lhs, const Variable &rhs);`
	`19`	`+ Variable operator /(const Variable &lhs, const Variable &rhs);`
	`20`	`+`
	`21`	`+ Variable negate(const Variable &input);`
	`22`	`+ Variable reciprocal(const Variable &input);`
`18`	`23`	`}`
`19`	`24`	`}`
Original file line number	Diff line number	Diff line change
`@@ -21,16 +21,16 @@ namespace af {`
`21`	`21`	`m_grad_func(nullptr)`
`22`	`22`	`{}`
`23`	`23`
`24`		`- Variable::Shared::Shared(af::array data, bool calc_grad) :`
	`24`	`+ Variable::Shared::Shared(const af::array &data, bool calc_grad) :`
`25`	`25`	`m_calc_grad(calc_grad),`
`26`	`26`	`m_data(data),`
`27`	`27`	`m_inputs(),`
`28`	`28`	`m_grads(),`
`29`	`29`	`m_grad_func(nullptr)`
`30`	`30`	`{}`
`31`	`31`
`32`		`- Variable::Shared::Shared(af::array data,`
`33`		`- std::vector<Variable> inputs,`
	`32`	`+ Variable::Shared::Shared(const af::array &data,`
	`33`	`+ const std::vector<Variable> &inputs,`
`34`	`34`	`GradFunc_t grad_func,`
`35`	`35`	`bool calc_grad) :`
`36`	`36`	`m_calc_grad(calc_grad),`
`@@ -45,13 +45,13 @@ namespace af {`
`45`	`45`	`{`
`46`	`46`	`}`
`47`	`47`
`48`		`- Variable::Variable(af::array data, bool calc_grad) :`
	`48`	`+ Variable::Variable(const af::array &data, bool calc_grad) :`
`49`	`49`	`m_shared(new Shared(data, calc_grad))`
`50`	`50`	`{}`
`51`	`51`
`52`		`- Variable::Variable(af::array data,`
`53`		`- std::vector<Variable> inputs,`
`54`		`- GradFunc_t grad_func) :`
	`52`	`+ Variable::Variable(const af::array &data,`
	`53`	`+ const std::vector<Variable> &inputs,`
	`54`	`+ GradFunc_t grad_func) :`
`55`	`55`	`m_shared(nullptr)`
`56`	`56`	`{`
`57`	`57`	`bool calc_grad = false;`
`@@ -81,7 +81,7 @@ namespace af {`
`81`	`81`	`return m_shared->m_grads[0];`
`82`	`82`	`}`
`83`	`83`
`84`		`- bool Variable::isCalcGrad()`
	`84`	`+ bool Variable::isCalcGrad() const`
`85`	`85`	`{`
`86`	`86`	`return m_shared->m_calc_grad;`
`87`	`87`	`}`
`@@ -96,7 +96,7 @@ namespace af {`
`96`	`96`	`}`
`97`	`97`	`}`
`98`	`98`
`99`		`- void Variable::addGrad(Variable child_grad)`
	`99`	`+ void Variable::addGrad(const Variable &child_grad)`
`100`	`100`	`{`
`101`	`101`	`if (m_shared->m_calc_grad) {`
`102`	`102`	`m_shared->m_grads.push_back(child_grad);`
`@@ -107,13 +107,17 @@ namespace af {`
`107`	`107`	`{`
`108`	`108`	`// Flag asking not to calculate gradients`
`109`	`109`	`if (!m_shared->m_calc_grad) return;`
`110`		`- Variable grad = m_shared->m_grads[0];`
`111`		`- for (unsigned i = 1; i < m_shared->m_grads.size(); i++) {`
`112`		`- grad = grad + m_shared->m_grads[i];`
	`110`	`+`
	`111`	`+ // Best not to evaluate the JIT immediately if theres only a single gradient`
	`112`	`+ if (m_shared->m_grads.size() > 1) {`
	`113`	`+ Variable grad = m_shared->m_grads[0];`
	`114`	`+ for (unsigned i = 1; i < m_shared->m_grads.size(); i++) {`
	`115`	`+ grad = grad + m_shared->m_grads[i];`
	`116`	`+ }`
	`117`	`+ grad.array().eval();`
	`118`	`+ m_shared->m_grads.clear();`
	`119`	`+ m_shared->m_grads.push_back(grad);`
`113`	`120`	`}`
`114`		`- grad.array().eval();`
`115`		`- m_shared->m_grads.clear();`
`116`		`- m_shared->m_grads.push_back(grad);`
`117`	`121`	`}`
`118`	`122`
`119`	`123`	`void Variable::calcGradInputs()`
`@@ -124,7 +128,7 @@ namespace af {`
`124`	`128`	`}`
`125`	`129`	`}`
`126`	`130`
`127`		`- void Variable::backward(Variable grad)`
	`131`	`+ void Variable::backward(const Variable &grad)`
`128`	`132`	`{`
`129`	`133`	`this->addGrad(grad);`
`130`	`134`	`Variable::DAG_t dag = this->build();`