From a05206c077561095bf35390154826fe30073d348 Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Mon, 12 Dec 2016 18:00:47 +0530
Subject: [PATCH 01/82] Modify CPU backend device management to be thread safe

---
 src/backend/cpu/platform.cpp | 174 ++++++++++++-----------------------
 src/backend/cpu/platform.hpp | 130 +++++++++++++++++++++++---
 2 files changed, 175 insertions(+), 129 deletions(-)

diff --git a/src/backend/cpu/platform.cpp b/src/backend/cpu/platform.cpp
index 5afd8316bf..105346ea37 100644
--- a/src/backend/cpu/platform.cpp
+++ b/src/backend/cpu/platform.cpp
@@ -9,85 +9,15 @@
 
 #include <af/version.h>
 #include <platform.hpp>
-#include <sstream>
-#include <queue.hpp>
-#include <array>
-#include <algorithm>
-#include <iostream>
-#include <string>
 #include <defines.hpp>
 #include <version.hpp>
-#include <queue.hpp>
 #include <host_memory.hpp>
-#include <cctype>
-
-
-#if defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || defined(_M_IX86) || defined(_WIN64)
-#define CPUID_CAPABLE USE_CPUID
-#else
-#define CPUID_CAPABLE 0
-#endif
 
-#ifdef _WIN32
-#include <limits.h>
-#include <intrin.h>
-typedef unsigned __int32  uint32_t;
-#else
-#include <stdint.h>
-#endif
+#include <cctype>
+#include <sstream>
 
 using namespace std;
 
-#if CPUID_CAPABLE
-
-#define MAX_INTEL_TOP_LVL 4
-
-class CPUID {
-    uint32_t regs[4];
-
-    public:
-    explicit CPUID(unsigned funcId, unsigned subFuncId) {
-#ifdef _WIN32
-        __cpuidex((int *)regs, (int)funcId, (int)subFuncId);
-
-#else
-        asm volatile
-            ("cpuid" : "=a" (regs[0]), "=b" (regs[1]), "=c" (regs[2]), "=d" (regs[3])
-             : "a" (funcId), "c" (subFuncId));
-#endif
-    }
-
-    inline const uint32_t &EAX() const { return regs[0]; }
-    inline const uint32_t &EBX() const { return regs[1]; }
-    inline const uint32_t &ECX() const { return regs[2]; }
-    inline const uint32_t &EDX() const { return regs[3]; }
-};
-
-#endif
-
-class CPUInfo {
-    public:
-        CPUInfo();
-        string  vendor()   const { return mVendorId;   }
-        string  model()    const { return mModelName;  }
-        int     threads()  const { return mNumLogCpus; }
-
-    private:
-        // Bit positions for data extractions
-        static const uint32_t LVL_NUM   = 0x000000FF;
-        static const uint32_t LVL_TYPE  = 0x0000FF00;
-        static const uint32_t LVL_CORES = 0x0000FFFF;
-        static const uint32_t HTT_POS   = 0x10000000;
-
-        // Attributes
-        string mVendorId;
-        string mModelName;
-        int    mNumSMT;
-        int    mNumCores;
-        int    mNumLogCpus;
-        bool   mIsHTT;
-};
-
 #if !CPUID_CAPABLE
 
 CPUInfo::CPUInfo()
@@ -116,7 +46,9 @@ CPUInfo::CPUInfo()
     mVendorId += string((const char *)&cpuID0.ECX(), 4);
 
     string upVId = mVendorId;
+
     for_each(upVId.begin(), upVId.end(), [](char& in) { in = ::toupper(in); });
+
     // Get num of cores
     if (upVId.find("INTEL") != std::string::npos) {
         mVendorId = "Intel";
@@ -166,8 +98,8 @@ CPUInfo::CPUInfo()
             mNumCores = mNumLogCpus = 1;
         }
     } else {
-        mVendorId = "Unkown, probably ARM";
-        cout<< "Unexpected vendor id" <<endl;
+        mVendorId = "Unkown";
+        std::cout<< "Unexpected vendor id" << std::endl;
     }
     // Get processor brand string
     // This seems to be working for both Intel & AMD vendors
@@ -186,27 +118,6 @@ CPUInfo::CPUInfo()
 namespace cpu
 {
 
-unsigned getMaxJitSize()
-{
-    const int MAX_JIT_LEN = 100;
-
-    static int length = 0;
-    if (length == 0) {
-        std::string env_var = getEnvVar("AF_CPU_MAX_JIT_LEN");
-        if (!env_var.empty()) {
-            length = std::stoi(env_var);
-        } else {
-            length = MAX_JIT_LEN;
-        }
-    }
-    return length;
-}
-
-int getBackend()
-{
-    return AF_BACKEND_CPU;
-}
-
 static const std::string get_system(void)
 {
     std::string arch = (sizeof(void *) == 4) ? "32-bit " : "64-bit ";
@@ -230,15 +141,24 @@ static inline std::string &ltrim(std::string &s)
     return s;
 }
 
+int getBackend()
+{
+    return AF_BACKEND_CPU;
+}
+
 std::string getDeviceInfo()
 {
+    const CPUInfo cinfo = DeviceManager::getInstance().getCPUInfo();
+
     std::ostringstream info;
-    static CPUInfo cinfo;
 
     info << "ArrayFire v" << AF_VERSION
          << " (CPU, " << get_system() << ", build " << AF_REVISION << ")" << std::endl;
+
     std::string model = cinfo.model();
+
     size_t memMB = getDeviceMemorySize(getActiveDeviceId()) / 1048576;
+
     info << string("[0] ") << cinfo.vendor() <<": " << ltrim(model);
 
     if(memMB) info << ", " << memMB << " MB, ";
@@ -249,17 +169,19 @@ std::string getDeviceInfo()
     info << AF_COMPILER_STR;
 #endif
     info << std::endl;
+
     return info.str();
 }
 
 bool isDoubleSupported(int device)
 {
-    return true;
+    return DeviceManager::IS_DOUBLE_SUPPORTED;
 }
 
 void devprop(char* d_name, char* d_platform, char *d_toolkit, char* d_compute)
 {
-    static CPUInfo cinfo;
+    const CPUInfo cinfo = DeviceManager::getInstance().getCPUInfo();
+
     snprintf(d_name, 64, "%s", cinfo.vendor().c_str());
     snprintf(d_platform, 10, "CPU");
     // report the compiler for toolkit
@@ -267,25 +189,30 @@ void devprop(char* d_name, char* d_platform, char *d_toolkit, char* d_compute)
     snprintf(d_compute, 10, "%s", "0.0");
 }
 
-int getDeviceCount()
+unsigned getMaxJitSize()
 {
-    return 1;
-}
+    const int MAX_JIT_LEN = 100;
 
+    static int length = 0;
+    if (length == 0) {
+        std::string env_var = getEnvVar("AF_CPU_MAX_JIT_LEN");
+        if (!env_var.empty()) {
+            length = std::stoi(env_var);
+        } else {
+            length = MAX_JIT_LEN;
+        }
+    }
+    return length;
+}
 
-int setDevice(int device)
+int getDeviceCount()
 {
-    static bool flag;
-    if(!flag && device != 0) {
-        printf("WARNING af_set_device(device): device can only be 0 for CPU\n");
-        flag = 1;
-    }
-    return 0;
+    return DeviceManager::NUM_DEVICES;
 }
 
 int getActiveDeviceId()
 {
-    return 0;
+    return DeviceManager::ACTIVE_DEVICE_ID;
 }
 
 size_t getDeviceMemorySize(int device)
@@ -298,17 +225,31 @@ size_t getHostMemorySize()
     return common::getHostMemorySize();
 }
 
-static const int MAX_QUEUES = 1;
+int setDevice(int device)
+{
+    static bool flag = false;
+    if (!flag && device != 0) {
+#ifndef NDEBUG
+        std::cerr << "WARNING af_set_device(device): device can only be 0 for CPU\n";
+#endif
+        flag = true;
+    }
+    return 0;
+}
 
+queue& getQueue(int device)
+{
+    return DeviceManager::getInstance().queues[device];
+}
 
-queue& getQueue(int idx) {
-    static std::array<queue, MAX_QUEUES> queues;
-    return queues[idx];
+CPUInfo DeviceManager::getCPUInfo() const
+{
+    return cinfo;
 }
 
 void sync(int device)
 {
-    getQueue().sync();
+    getQueue(device).sync();
 }
 
 bool& evalFlag()
@@ -317,5 +258,10 @@ bool& evalFlag()
     return flag;
 }
 
+DeviceManager& DeviceManager::getInstance()
+{
+    static DeviceManager my_instance;
+    return my_instance;
+}
 
 }
diff --git a/src/backend/cpu/platform.hpp b/src/backend/cpu/platform.hpp
index edf7a072fc..066b711f88 100644
--- a/src/backend/cpu/platform.hpp
+++ b/src/backend/cpu/platform.hpp
@@ -9,34 +9,134 @@
 
 #pragma once
 
+#include <array>
+#include <algorithm>
+#include <iostream>
 #include <string>
 
-namespace cpu {
-    class queue;
+#include <queue.hpp>
 
-    int getBackend();
+#if defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || defined(_M_IX86) || defined(_WIN64)
+#define CPUID_CAPABLE USE_CPUID
+#else
+#define CPUID_CAPABLE 0
+#endif
 
-    std::string getDeviceInfo();
+#ifdef _WIN32
+#include <limits.h>
+#include <intrin.h>
+typedef unsigned __int32  uint32_t;
+#else
+#include <stdint.h>
+#endif
 
-    bool isDoubleSupported(int device);
+#if CPUID_CAPABLE
 
-    void devprop(char* d_name, char* d_platform, char *d_toolkit, char* d_compute);
+#define MAX_INTEL_TOP_LVL 4
 
-    int getDeviceCount();
+class CPUID {
+    uint32_t regs[4];
 
-    int setDevice(int device);
+    public:
+    explicit CPUID(unsigned funcId, unsigned subFuncId) {
+#ifdef _WIN32
+        __cpuidex((int *)regs, (int)funcId, (int)subFuncId);
 
-    int getActiveDeviceId();
+#else
+        asm volatile
+            ("cpuid" : "=a" (regs[0]), "=b" (regs[1]), "=c" (regs[2]), "=d" (regs[3])
+             : "a" (funcId), "c" (subFuncId));
+#endif
+    }
 
-    size_t getDeviceMemorySize(int device);
+    inline const uint32_t &EAX() const { return regs[0]; }
+    inline const uint32_t &EBX() const { return regs[1]; }
+    inline const uint32_t &ECX() const { return regs[2]; }
+    inline const uint32_t &EDX() const { return regs[3]; }
+};
 
-    size_t getHostMemorySize();
+#endif
 
-    void sync(int device);
+class CPUInfo {
+    public:
+        CPUInfo();
+        std::string  vendor() const { return mVendorId;   }
+        std::string  model()  const { return mModelName;  }
+        int threads() const { return mNumLogCpus; }
 
-    queue& getQueue(int idx = 0);
+    private:
+        // Bit positions for data extractions
+        static const uint32_t LVL_NUM   = 0x000000FF;
+        static const uint32_t LVL_TYPE  = 0x0000FF00;
+        static const uint32_t LVL_CORES = 0x0000FFFF;
+        static const uint32_t HTT_POS   = 0x10000000;
 
-    unsigned getMaxJitSize();
+        // Attributes
+        std::string mVendorId;
+        std::string mModelName;
+        int    mNumSMT;
+        int    mNumCores;
+        int    mNumLogCpus;
+        bool   mIsHTT;
+};
+
+namespace cpu
+{
+
+int getBackend();
+
+std::string getDeviceInfo();
+
+bool isDoubleSupported(int device);
+
+void devprop(char* d_name, char* d_platform, char *d_toolkit, char* d_compute);
+
+unsigned getMaxJitSize();
+
+int getDeviceCount();
+
+int getActiveDeviceId();
+
+size_t getDeviceMemorySize(int device);
+
+size_t getHostMemorySize();
+
+int setDevice(int device);
+
+queue& getQueue(int device=0);
+
+void sync(int device);
+
+bool& evalFlag();
+
+class DeviceManager
+{
+    public:
+        static const int MAX_QUEUES = 1;
+        static const int NUM_DEVICES = 1;
+        static const int ACTIVE_DEVICE_ID = 0;
+        static const bool IS_DOUBLE_SUPPORTED = true;
+
+        static DeviceManager& getInstance();
+
+        friend queue& getQueue(int device);
+
+        CPUInfo getCPUInfo() const;
+
+    private:
+        DeviceManager() {
+        }
+
+        // Following two declarations are required to
+        // avoid copying accidental copy/assignment
+        // of instance returned by getInstance to other
+        // variables
+        DeviceManager(DeviceManager const&);
+        void operator=(DeviceManager const&);
+
+        // Attributes
+        const CPUInfo cinfo;
+        std::array<queue, MAX_QUEUES> queues;
+};
 
-    bool& evalFlag();
 }

From ddf6fde42ec42141f7e32e89b26acbfadb352f07 Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Thu, 22 Dec 2016 00:11:37 +0530
Subject: [PATCH 02/82] Merge cuda::InteropManager into cuda::DeviceManager

Access to InteropManager has to go through device manager now.
---
 src/backend/cuda/hist_graphics.cpp  |  6 ++---
 src/backend/cuda/image.cpp          |  4 ++--
 src/backend/cuda/interopManager.cpp | 28 ----------------------
 src/backend/cuda/interopManager.hpp |  5 +---
 src/backend/cuda/platform.cpp       | 36 ++++++++++++++++++++++++++++-
 src/backend/cuda/platform.hpp       | 12 ++++++++++
 src/backend/cuda/plot.cpp           |  6 ++---
 src/backend/cuda/surface.cpp        |  6 ++---
 src/backend/cuda/vector_field.cpp   |  6 ++---
 9 files changed, 62 insertions(+), 47 deletions(-)

diff --git a/src/backend/cuda/hist_graphics.cpp b/src/backend/cuda/hist_graphics.cpp
index 4e9017da88..d207961f59 100644
--- a/src/backend/cuda/hist_graphics.cpp
+++ b/src/backend/cuda/hist_graphics.cpp
@@ -9,11 +9,11 @@
 
 #if defined (WITH_GRAPHICS)
 
-#include <interopManager.hpp>
 #include <Array.hpp>
 #include <hist_graphics.hpp>
 #include <err_cuda.hpp>
 #include <debug_cuda.hpp>
+#include <interopManager.hpp>
 
 namespace cuda
 {
@@ -22,10 +22,10 @@ using namespace gl;
 template<typename T>
 void copy_histogram(const Array<T> &data, const forge::Histogram* hist)
 {
-    if(InteropManager::checkGraphicsInteropCapability()) {
+    if(DeviceManager::checkGraphicsInteropCapability()) {
         const T *d_P = data.get();
 
-        InteropManager& intrpMngr = InteropManager::getInstance();
+        InteropManager& intrpMngr = DeviceManager::getInstance().getGfxInteropManager();
 
         cudaGraphicsResource_t *resources = intrpMngr.getBufferResource(hist);
         // Map resource. Copy data to VBO. Unmap resource.
diff --git a/src/backend/cuda/image.cpp b/src/backend/cuda/image.cpp
index 2303184daf..ca87d3129f 100644
--- a/src/backend/cuda/image.cpp
+++ b/src/backend/cuda/image.cpp
@@ -27,8 +27,8 @@ using namespace gl;
 template<typename T>
 void copy_image(const Array<T> &in, const forge::Image* image)
 {
-    if(InteropManager::checkGraphicsInteropCapability()) {
-        InteropManager& intrpMngr = InteropManager::getInstance();
+    if(DeviceManager::checkGraphicsInteropCapability()) {
+        InteropManager& intrpMngr = DeviceManager::getInstance().getGfxInteropManager();
 
         cudaGraphicsResource_t *resources = intrpMngr.getBufferResource(image);
 
diff --git a/src/backend/cuda/interopManager.cpp b/src/backend/cuda/interopManager.cpp
index 077e4ac703..a8339a52f4 100644
--- a/src/backend/cuda/interopManager.cpp
+++ b/src/backend/cuda/interopManager.cpp
@@ -50,12 +50,6 @@ InteropManager::~InteropManager()
     }
 }
 
-InteropManager& InteropManager::getInstance()
-{
-    static InteropManager my_instance;
-    return my_instance;
-}
-
 interop_t& InteropManager::getDeviceMap(int device)
 {
     return (device == -1) ? interop_maps[getActiveDeviceId()] : interop_maps[device];
@@ -172,28 +166,6 @@ CGR_t* InteropManager::getBufferResource(const forge::VectorField* key)
     return &i_map[key_value].front();
 }
 
-bool InteropManager::checkGraphicsInteropCapability()
-{
-    static bool run_once = true;
-    static bool capable  = true;
-
-    if(run_once) {
-        unsigned int pCudaEnabledDeviceCount = 0;
-        int pCudaGraphicsEnabledDeviceIds = 0;
-        cudaGetLastError(); // Reset Errors
-        cudaError_t err = cudaGLGetDevices(&pCudaEnabledDeviceCount, &pCudaGraphicsEnabledDeviceIds, getDeviceCount(), cudaGLDeviceListAll);
-        if(err == 63) { // OS Support Failure - Happens when devices are only Tesla
-            capable = false;
-            printf("Warning: No CUDA Device capable of CUDA-OpenGL. CUDA-OpenGL Interop will use CPU fallback.\n");
-            printf("Corresponding CUDA Error (%d): %s.\n", err, cudaGetErrorString(err));
-            printf("This may happen if all CUDA Devices are in TCC Mode and/or not connected to a display.\n");
-        }
-        cudaGetLastError(); // Reset Errors
-        run_once = false;
-    }
-    return capable;
-}
-
 }
 
 #endif
diff --git a/src/backend/cuda/interopManager.hpp b/src/backend/cuda/interopManager.hpp
index deb4f126e9..38ecee2400 100644
--- a/src/backend/cuda/interopManager.hpp
+++ b/src/backend/cuda/interopManager.hpp
@@ -52,9 +52,7 @@ class InteropManager
         interop_t interop_maps[DeviceManager::MAX_DEVICES];
 
     public:
-        static InteropManager& getInstance();
-        static bool checkGraphicsInteropCapability();
-
+        InteropManager() {}
         ~InteropManager();
         CGR_t* getBufferResource(const forge::Image       *handle);
         CGR_t* getBufferResource(const forge::Plot        *handle);
@@ -63,7 +61,6 @@ class InteropManager
         CGR_t* getBufferResource(const forge::VectorField *handle);
 
     protected:
-        InteropManager() {}
         InteropManager(InteropManager const&);
         void operator=(InteropManager const&);
         interop_t& getDeviceMap(int device = -1); // default will return current device
diff --git a/src/backend/cuda/platform.cpp b/src/backend/cuda/platform.cpp
index 8a13993a12..752ea71c96 100644
--- a/src/backend/cuda/platform.cpp
+++ b/src/backend/cuda/platform.cpp
@@ -24,6 +24,7 @@
 #include <err_cuda.hpp>
 #include <util.hpp>
 #include <host_memory.hpp>
+#include <interopManager.hpp>
 
 using namespace std;
 
@@ -350,14 +351,47 @@ cudaDeviceProp getDeviceProp(int device)
 ///////////////////////////////////////////////////////////////////////////
 // DeviceManager Class Functions
 ///////////////////////////////////////////////////////////////////////////
+bool DeviceManager::checkGraphicsInteropCapability()
+{
+    static bool run_once = true;
+    static bool capable  = true;
+
+    if(run_once) {
+        unsigned int pCudaEnabledDeviceCount = 0;
+        int pCudaGraphicsEnabledDeviceIds = 0;
+        cudaGetLastError(); // Reset Errors
+        cudaError_t err = cudaGLGetDevices(&pCudaEnabledDeviceCount, &pCudaGraphicsEnabledDeviceIds, getDeviceCount(), cudaGLDeviceListAll);
+        if(err == 63) { // OS Support Failure - Happens when devices are only Tesla
+            capable = false;
+            printf("Warning: No CUDA Device capable of CUDA-OpenGL. CUDA-OpenGL Interop will use CPU fallback.\n");
+            printf("Corresponding CUDA Error (%d): %s.\n", err, cudaGetErrorString(err));
+            printf("This may happen if all CUDA Devices are in TCC Mode and/or not connected to a display.\n");
+        }
+        cudaGetLastError(); // Reset Errors
+        run_once = false;
+    }
+
+    return capable;
+}
+
 DeviceManager& DeviceManager::getInstance()
 {
     static DeviceManager my_instance;
     return my_instance;
 }
 
+DeviceManager::~DeviceManager()
+{
+    if (gfxManager) delete gfxManager;
+}
+
+InteropManager& DeviceManager::getGfxInteropManager()
+{
+    return *gfxManager;
+}
+
 DeviceManager::DeviceManager()
-    : cuDevices(0), activeDev(0), nDevices(0)
+    : cuDevices(0), activeDev(0), nDevices(0), gfxManager(new InteropManager())
 {
     CUDA_CHECK(cudaGetDeviceCount(&nDevices));
     if (nDevices == 0)
diff --git a/src/backend/cuda/platform.hpp b/src/backend/cuda/platform.hpp
index fa73ccf5e0..ca7ba418bd 100644
--- a/src/backend/cuda/platform.hpp
+++ b/src/backend/cuda/platform.hpp
@@ -66,13 +66,21 @@ struct cudaDevice_t {
 
 bool& evalFlag();
 
+class InteropManager;
+
 class DeviceManager
 {
     public:
         static const unsigned MAX_DEVICES = 16;
 
+        static bool checkGraphicsInteropCapability();
+
         static DeviceManager& getInstance();
 
+        ~DeviceManager();
+
+        InteropManager& getGfxInteropManager();
+
         friend std::string getDeviceInfo(int device);
 
         friend std::string getPlatformInfo();
@@ -119,6 +127,10 @@ class DeviceManager
         int activeDev;
         int nDevices;
         cudaStream_t streams[MAX_DEVICES];
+        //FIXME: Once C++11 has been enabled in CUDA backend
+        //shift to use of smart pointer. In this, unique_ptr
+        //should be good
+        InteropManager* gfxManager;
 };
 
 }
diff --git a/src/backend/cuda/plot.cpp b/src/backend/cuda/plot.cpp
index 6c1aad70af..40ae7b4574 100644
--- a/src/backend/cuda/plot.cpp
+++ b/src/backend/cuda/plot.cpp
@@ -9,7 +9,6 @@
 
 #if defined (WITH_GRAPHICS)
 
-#include <interopManager.hpp>
 #include <Array.hpp>
 #include <plot.hpp>
 #include <err_cuda.hpp>
@@ -17,6 +16,7 @@
 #include <join.hpp>
 #include <reduce.hpp>
 #include <reorder.hpp>
+#include <interopManager.hpp>
 
 using af::dim4;
 
@@ -27,10 +27,10 @@ using namespace gl;
 template<typename T>
 void copy_plot(const Array<T> &P, forge::Plot* plot)
 {
-    if(InteropManager::checkGraphicsInteropCapability()) {
+    if(DeviceManager::checkGraphicsInteropCapability()) {
         const T *d_P = P.get();
 
-        InteropManager& intrpMngr = InteropManager::getInstance();
+        InteropManager& intrpMngr = DeviceManager::getInstance().getGfxInteropManager();
 
         cudaGraphicsResource_t *resources = intrpMngr.getBufferResource(plot);
         // Map resource. Copy data to VBO. Unmap resource.
diff --git a/src/backend/cuda/surface.cpp b/src/backend/cuda/surface.cpp
index 6d84d714b6..18385decd6 100644
--- a/src/backend/cuda/surface.cpp
+++ b/src/backend/cuda/surface.cpp
@@ -9,7 +9,6 @@
 
 #if defined (WITH_GRAPHICS)
 
-#include <interopManager.hpp>
 #include <Array.hpp>
 #include <surface.hpp>
 #include <err_cuda.hpp>
@@ -17,6 +16,7 @@
 #include <join.hpp>
 #include <reduce.hpp>
 #include <reorder.hpp>
+#include <interopManager.hpp>
 
 using af::dim4;
 
@@ -27,10 +27,10 @@ using namespace gl;
 template<typename T>
 void copy_surface(const Array<T> &P, forge::Surface* surface)
 {
-    if(InteropManager::checkGraphicsInteropCapability()) {
+    if(DeviceManager::checkGraphicsInteropCapability()) {
         const T *d_P = P.get();
 
-        InteropManager& intrpMngr = InteropManager::getInstance();
+        InteropManager& intrpMngr = DeviceManager::getInstance().getGfxInteropManager();
 
         cudaGraphicsResource_t *resources = intrpMngr.getBufferResource(surface);
         // Map resource. Copy data to VBO. Unmap resource.
diff --git a/src/backend/cuda/vector_field.cpp b/src/backend/cuda/vector_field.cpp
index 7ce587f199..6fa4dd9fee 100644
--- a/src/backend/cuda/vector_field.cpp
+++ b/src/backend/cuda/vector_field.cpp
@@ -9,11 +9,11 @@
 
 #if defined (WITH_GRAPHICS)
 
-#include <interopManager.hpp>
 #include <Array.hpp>
 #include <vector_field.hpp>
 #include <err_cuda.hpp>
 #include <debug_cuda.hpp>
+#include <interopManager.hpp>
 
 using af::dim4;
 
@@ -25,8 +25,8 @@ template<typename T>
 void copy_vector_field(const Array<T> &points, const Array<T> &directions,
                        forge::VectorField* vector_field)
 {
-    if(InteropManager::checkGraphicsInteropCapability()) {
-        InteropManager& intrpMngr = InteropManager::getInstance();
+    if(DeviceManager::checkGraphicsInteropCapability()) {
+        InteropManager& intrpMngr = DeviceManager::getInstance().getGfxInteropManager();
 
         cudaGraphicsResource_t *resources = intrpMngr.getBufferResource(vector_field);
 

From 6ff8d910914581369a3ede508465716cd5fd3fbe Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Thu, 22 Dec 2016 17:31:12 +0530
Subject: [PATCH 03/82] Merge cufft plan cacher into cuda::DeviceManager

Access to fft plan cacher singleton has to go through
device manager now.
---
 src/backend/cuda/cufftManager.cpp       | 149 ++++++++++++++++++++++
 src/backend/cuda/cufftManager.hpp       | 121 ++++++++++++++++++
 src/backend/cuda/err_cufft.hpp          |  92 --------------
 src/backend/cuda/fft.cpp                | 159 +-----------------------
 src/backend/cuda/kernel/fftconvolve.hpp |   2 -
 src/backend/cuda/platform.cpp           |   5 +
 src/backend/cuda/platform.hpp           |   7 ++
 7 files changed, 287 insertions(+), 248 deletions(-)
 create mode 100644 src/backend/cuda/cufftManager.cpp
 create mode 100644 src/backend/cuda/cufftManager.hpp
 delete mode 100644 src/backend/cuda/err_cufft.hpp

diff --git a/src/backend/cuda/cufftManager.cpp b/src/backend/cuda/cufftManager.cpp
new file mode 100644
index 0000000000..f3aea5f45a
--- /dev/null
+++ b/src/backend/cuda/cufftManager.cpp
@@ -0,0 +1,149 @@
+/*******************************************************
+ * Copyright (c) 2016, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <cuda.h> // Need this for CUDA_VERSION
+#include <platform.hpp>
+#include <memory.hpp>
+
+namespace cufft
+{
+
+const char * _cufftGetResultString(cufftResult res)
+{
+    switch (res)
+    {
+        case CUFFT_SUCCESS:
+            return "cuFFT: success";
+
+        case CUFFT_INVALID_PLAN:
+            return "cuFFT: invalid plan handle passed";
+
+        case CUFFT_ALLOC_FAILED:
+            return "cuFFT: resources allocation failed";
+
+        case CUFFT_INVALID_TYPE:
+            return "cuFFT: invalid type (deprecated)";
+
+        case CUFFT_INVALID_VALUE:
+            return "cuFFT: invalid parameters passed to cuFFT API";
+
+        case CUFFT_INTERNAL_ERROR:
+            return "cuFFT: internal error detected using cuFFT";
+
+        case CUFFT_EXEC_FAILED:
+            return "cuFFT: FFT execution failed";
+
+        case CUFFT_SETUP_FAILED:
+            return "cuFFT: library initialization failed";
+
+        case CUFFT_INVALID_SIZE:
+            return "cuFFT: invalid size parameters passed";
+
+        case CUFFT_UNALIGNED_DATA:
+            return "cuFFT: unaligned data (deprecated)";
+
+        case CUFFT_INCOMPLETE_PARAMETER_LIST:
+            return "cuFFT: call is missing parameters";
+
+        case CUFFT_INVALID_DEVICE:
+            return "cuFFT: plan execution different than plan creation";
+
+        case CUFFT_PARSE_ERROR:
+            return "cuFFT: plan parse error";
+
+        case CUFFT_NO_WORKSPACE:
+            return "cuFFT: no workspace provided";
+
+#if CUDA_VERSION >= 6050
+        case CUFFT_NOT_IMPLEMENTED:
+            return "cuFFT: not implemented";
+
+        case CUFFT_LICENSE_ERROR:
+            return "cuFFT: license error";
+#endif
+#if CUDA_VERSION >= 8000
+        case CUFFT_NOT_SUPPORTED:
+            return "cuFFT: not supported";
+#endif
+    }
+
+    return "cuFFT: unknown error";
+}
+
+void findPlan(cufftHandle &plan, int rank, int *n,
+              int *inembed, int istride, int idist,
+              int *onembed, int ostride, int odist,
+              cufftType type, int batch)
+{
+    // create the key string
+    char key_str_temp[64];
+    sprintf(key_str_temp, "%d:", rank);
+
+    std::string key_string(key_str_temp);
+
+    for(int r=0; r<rank; ++r) {
+        sprintf(key_str_temp, "%d:", n[r]);
+        key_string.append(std::string(key_str_temp));
+    }
+
+    if (inembed!=NULL) {
+        for(int r=0; r<rank; ++r) {
+            sprintf(key_str_temp, "%d:", inembed[r]);
+            key_string.append(std::string(key_str_temp));
+        }
+        sprintf(key_str_temp, "%d:%d:", istride, idist);
+        key_string.append(std::string(key_str_temp));
+    }
+
+    if (onembed!=NULL) {
+        for(int r=0; r<rank; ++r) {
+            sprintf(key_str_temp, "%d:", onembed[r]);
+            key_string.append(std::string(key_str_temp));
+        }
+        sprintf(key_str_temp, "%d:%d:", ostride, odist);
+        key_string.append(std::string(key_str_temp));
+    }
+
+    sprintf(key_str_temp, "%d:%d", (int)type, batch);
+    key_string.append(std::string(key_str_temp));
+
+    // find the matching plan_index in the array cuFFTPlanner::mKeys
+    cuFFTPlanner &planner = cuda::DeviceManager::getInstance().getcufftPlanManager();
+
+    int planIndex = planner.findIfPlanExists(key_string);
+
+    // if found a valid plan, return it
+    if (planIndex!=-1) {
+        plan = planner.getPlan(planIndex);
+        return;
+    }
+
+    cufftHandle temp;
+    cufftResult res = cufftPlanMany(&temp, rank, n,
+                                    inembed, istride, idist,
+                                    onembed, ostride, odist,
+                                    type, batch);
+
+    // If plan creation fails, clean up the memory we hold on to and try again
+    if (res != CUFFT_SUCCESS) {
+        cuda::garbageCollect();
+        CUFFT_CHECK(cufftPlanMany(&temp, rank, n,
+                                  inembed, istride, idist,
+                                  onembed, ostride, odist,
+                                  type, batch));
+    }
+
+    plan = temp;
+    cufftSetStream(plan, cuda::getStream(cuda::getActiveDeviceId()));
+
+    // push the plan into plan cache
+    planner.pushPlan(key_string, plan);
+}
+
+}
diff --git a/src/backend/cuda/cufftManager.hpp b/src/backend/cuda/cufftManager.hpp
new file mode 100644
index 0000000000..2d78f891c2
--- /dev/null
+++ b/src/backend/cuda/cufftManager.hpp
@@ -0,0 +1,121 @@
+/*******************************************************
+ * Copyright (c) 2016, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+#include <err_common.hpp>
+#include <cufft.h>
+
+#include <cstdio>
+#include <deque>
+#include <string>
+#include <utility>
+
+namespace cuda
+{
+class DeviceManager;
+}
+
+namespace cufft
+{
+
+typedef std::pair<std::string, cufftHandle> FFTPlanPair;
+typedef std::deque<FFTPlanPair> FFTPlanCache;
+
+const char * _cufftGetResultString(cufftResult res);
+
+void findPlan(cufftHandle &plan, int rank, int *n,
+              int *inembed, int istride, int idist,
+              int *onembed, int ostride, int odist,
+              cufftType type, int batch);
+
+// cuFFTPlanner caches fft plans
+//
+// new plan |--> IF number of plans cached is at limit, pop the least used entry and push new plan.
+//          |
+//          |--> ELSE just push the plan
+// existing plan -> reuse a plan
+class cuFFTPlanner
+{
+    friend class cuda::DeviceManager;
+
+    friend void findPlan(cufftHandle &plan, int rank, int *n,
+                         int *inembed, int istride, int idist,
+                         int *onembed, int ostride, int odist,
+                         cufftType type, int batch);
+
+    public:
+        inline void setMaxCacheSize(size_t size) {
+            mCache.resize(size, FFTPlanPair(std::string(""), 0));
+        }
+
+        inline size_t getMaxCacheSize() const {
+            return mMaxCacheSize;
+        }
+
+        inline cufftHandle getPlan(int index) const {
+            return mCache[index].second;
+        }
+
+        // iterates through plan cache from front to back
+        // of the cache(queue)
+        int findIfPlanExists(std::string keyString) const {
+            int retVal = -1;
+            for(uint i=0; i<mCache.size(); ++i) {
+                if (keyString == mCache[i].first) {
+                    retVal = i;
+                }
+            }
+            return retVal;
+        }
+
+        // pops plan from the back of cache(queue)
+        void popPlan() {
+            if (!mCache.empty()) {
+                // destroy the cufft plan associated with the
+                // least recently used plan
+                cufftDestroy(mCache.back().second);
+                // now pop the entry from cache
+                mCache.pop_back();
+            }
+        }
+
+        // pushes plan to the front of cache(queue)
+        void pushPlan(std::string keyString, cufftHandle plan) {
+            if (mCache.size()>mMaxCacheSize) {
+                popPlan();
+            }
+            mCache.push_front(FFTPlanPair(keyString, plan));
+        }
+
+    private:
+        cuFFTPlanner() : mMaxCacheSize(5) {}
+        cuFFTPlanner(cuFFTPlanner const&);
+        void operator=(cuFFTPlanner const&);
+
+        size_t       mMaxCacheSize;
+        FFTPlanCache mCache;
+};
+
+}
+
+#define CUFFT_CHECK(fn) do {                        \
+        cufftResult _cufft_res = fn;                \
+        if (_cufft_res != CUFFT_SUCCESS) {          \
+            char cufft_res_msg[1024];               \
+            snprintf(cufft_res_msg,                 \
+                     sizeof(cufft_res_msg),         \
+                     "cuFFT Error (%d): %s\n",      \
+                     (int)(_cufft_res),             \
+                     cufft::_cufftGetResultString(  \
+                         _cufft_res));              \
+                                                    \
+            AF_ERROR(cufft_res_msg,                 \
+                     AF_ERR_INTERNAL);              \
+        }                                           \
+    } while(0)
diff --git a/src/backend/cuda/err_cufft.hpp b/src/backend/cuda/err_cufft.hpp
deleted file mode 100644
index 99fcd97cee..0000000000
--- a/src/backend/cuda/err_cufft.hpp
+++ /dev/null
@@ -1,92 +0,0 @@
-/*******************************************************
- * Copyright (c) 2014, ArrayFire
- * All rights reserved.
- *
- * This file is distributed under 3-clause BSD license.
- * The complete license agreement can be obtained at:
- * http://arrayfire.com/licenses/BSD-3-Clause
- ********************************************************/
-
-#pragma once
-#include <stdio.h>
-#include <err_common.hpp>
-#include <cuda.h> // Need this for CUDA_VERSION
-#include <cufft.h>
-
-static const char * _cufftGetResultString(cufftResult res)
-{
-    switch (res)
-    {
-        case CUFFT_SUCCESS:
-            return "cuFFT: success";
-
-        case CUFFT_INVALID_PLAN:
-            return "cuFFT: invalid plan handle passed";
-
-        case CUFFT_ALLOC_FAILED:
-            return "cuFFT: resources allocation failed";
-
-        case CUFFT_INVALID_TYPE:
-            return "cuFFT: invalid type (deprecated)";
-
-        case CUFFT_INVALID_VALUE:
-            return "cuFFT: invalid parameters passed to cuFFT API";
-
-        case CUFFT_INTERNAL_ERROR:
-            return "cuFFT: internal error detected using cuFFT";
-
-        case CUFFT_EXEC_FAILED:
-            return "cuFFT: FFT execution failed";
-
-        case CUFFT_SETUP_FAILED:
-            return "cuFFT: library initialization failed";
-
-        case CUFFT_INVALID_SIZE:
-            return "cuFFT: invalid size parameters passed";
-
-        case CUFFT_UNALIGNED_DATA:
-            return "cuFFT: unaligned data (deprecated)";
-
-        case CUFFT_INCOMPLETE_PARAMETER_LIST:
-            return "cuFFT: call is missing parameters";
-
-        case CUFFT_INVALID_DEVICE:
-            return "cuFFT: plan execution different than plan creation";
-
-        case CUFFT_PARSE_ERROR:
-            return "cuFFT: plan parse error";
-
-        case CUFFT_NO_WORKSPACE:
-            return "cuFFT: no workspace provided";
-
-#if CUDA_VERSION >= 6050
-        case CUFFT_NOT_IMPLEMENTED:
-            return "cuFFT: not implemented";
-
-        case CUFFT_LICENSE_ERROR:
-            return "cuFFT: license error";
-#endif
-#if CUDA_VERSION >= 8000
-        case CUFFT_NOT_SUPPORTED:
-            return "cuFFT: not supported";
-#endif
-    }
-
-    return "cuFFT: unknown error";
-}
-
-#define CUFFT_CHECK(fn) do {                    \
-        cufftResult _cufft_res = fn;            \
-        if (_cufft_res != CUFFT_SUCCESS) {      \
-            char cufft_res_msg[1024];           \
-            snprintf(cufft_res_msg,             \
-                     sizeof(cufft_res_msg),     \
-                     "cuFFT Error (%d): %s\n",  \
-                     (int)(_cufft_res),         \
-                     _cufftGetResultString(     \
-                         _cufft_res));          \
-                                                \
-            AF_ERROR(cufft_res_msg,             \
-                     AF_ERR_INTERNAL);          \
-        }                                       \
-    } while(0)
diff --git a/src/backend/cuda/fft.cpp b/src/backend/cuda/fft.cpp
index 30e61084ce..b2b36c90e1 100644
--- a/src/backend/cuda/fft.cpp
+++ b/src/backend/cuda/fft.cpp
@@ -12,13 +12,8 @@
 #include <copy.hpp>
 #include <fft.hpp>
 #include <debug_cuda.hpp>
-#include <err_cufft.hpp>
-#include <cufft.h>
+#include <cufftManager.hpp>
 #include <math.hpp>
-#include <string>
-#include <deque>
-#include <utility>
-#include <cstdio>
 #include <memory.hpp>
 
 using af::dim4;
@@ -27,153 +22,9 @@ using std::string;
 namespace cuda
 {
 
-typedef std::pair<string, cufftHandle> FFTPlanPair;
-typedef std::deque<FFTPlanPair> FFTPlanCache;
-
-// cuFFTPlanner caches fft plans
-//
-// new plan |--> IF number of plans cached is at limit, pop the least used entry and push new plan.
-//          |
-//          |--> ELSE just push the plan
-// existing plan -> reuse a plan
-class cuFFTPlanner
-{
-    friend void find_cufft_plan(cufftHandle &plan, int rank, int *n,
-                                int *inembed, int istride, int idist,
-                                int *onembed, int ostride, int odist,
-                                cufftType type, int batch);
-
-    public:
-        static cuFFTPlanner& getInstance() {
-            static cuFFTPlanner instances[cuda::DeviceManager::MAX_DEVICES];
-            return instances[cuda::getActiveDeviceId()];
-        }
-
-        inline void setMaxCacheSize(size_t size) {
-            mCache.resize(size, FFTPlanPair(std::string(""), 0));
-        }
-
-        inline size_t getMaxCacheSize() const {
-            return mMaxCacheSize;
-        }
-
-        inline cufftHandle getPlan(int index) const {
-            return mCache[index].second;
-        }
-
-        // iterates through plan cache from front to back
-        // of the cache(queue)
-        int findIfPlanExists(std::string keyString) const {
-            int retVal = -1;
-            for(uint i=0; i<mCache.size(); ++i) {
-                if (keyString == mCache[i].first) {
-                    retVal = i;
-                }
-            }
-            return retVal;
-        }
-
-        // pops plan from the back of cache(queue)
-        void popPlan() {
-            if (!mCache.empty()) {
-                // destroy the cufft plan associated with the
-                // least recently used plan
-                cufftDestroy(mCache.back().second);
-                // now pop the entry from cache
-                mCache.pop_back();
-            }
-        }
-
-        // pushes plan to the front of cache(queue)
-        void pushPlan(std::string keyString, cufftHandle plan) {
-            if (mCache.size()>mMaxCacheSize) {
-                popPlan();
-            }
-            mCache.push_front(FFTPlanPair(keyString, plan));
-        }
-
-    private:
-        cuFFTPlanner() : mMaxCacheSize(5) {}
-        cuFFTPlanner(cuFFTPlanner const&);
-        void operator=(cuFFTPlanner const&);
-
-        size_t       mMaxCacheSize;
-        FFTPlanCache mCache;
-};
-
-void find_cufft_plan(cufftHandle &plan, int rank, int *n,
-                     int *inembed, int istride, int idist,
-                     int *onembed, int ostride, int odist,
-                     cufftType type, int batch)
-{
-    // create the key string
-    char key_str_temp[64];
-    sprintf(key_str_temp, "%d:", rank);
-
-    string key_string(key_str_temp);
-
-    for(int r=0; r<rank; ++r) {
-        sprintf(key_str_temp, "%d:", n[r]);
-        key_string.append(std::string(key_str_temp));
-    }
-
-    if (inembed!=NULL) {
-        for(int r=0; r<rank; ++r) {
-            sprintf(key_str_temp, "%d:", inembed[r]);
-            key_string.append(std::string(key_str_temp));
-        }
-        sprintf(key_str_temp, "%d:%d:", istride, idist);
-        key_string.append(std::string(key_str_temp));
-    }
-
-    if (onembed!=NULL) {
-        for(int r=0; r<rank; ++r) {
-            sprintf(key_str_temp, "%d:", onembed[r]);
-            key_string.append(std::string(key_str_temp));
-        }
-        sprintf(key_str_temp, "%d:%d:", ostride, odist);
-        key_string.append(std::string(key_str_temp));
-    }
-
-    sprintf(key_str_temp, "%d:%d", (int)type, batch);
-    key_string.append(std::string(key_str_temp));
-
-    // find the matching plan_index in the array cuFFTPlanner::mKeys
-    cuFFTPlanner &planner = cuFFTPlanner::getInstance();
-
-    int planIndex = planner.findIfPlanExists(key_string);
-
-    // if found a valid plan, return it
-    if (planIndex!=-1) {
-        plan = planner.getPlan(planIndex);
-        return;
-    }
-
-    cufftHandle temp;
-    cufftResult res = cufftPlanMany(&temp, rank, n,
-                                    inembed, istride, idist,
-                                    onembed, ostride, odist,
-                                    type, batch);
-
-    // If plan creation fails, clean up the memory we hold on to and try again
-    if (res != CUFFT_SUCCESS) {
-        garbageCollect();
-        CUFFT_CHECK(cufftPlanMany(&temp, rank, n,
-                                  inembed, istride, idist,
-                                  onembed, ostride, odist,
-                                  type, batch));
-    }
-
-    plan = temp;
-    cufftSetStream(plan, cuda::getStream(cuda::getActiveDeviceId()));
-
-    // push the plan into plan cache
-    planner.pushPlan(key_string, plan);
-}
-
 void setFFTPlanCacheSize(size_t numPlans)
 {
-    cuFFTPlanner::getInstance().setMaxCacheSize(numPlans);
+    DeviceManager::getInstance().getcufftPlanManager().setMaxCacheSize(numPlans);
 }
 
 template<typename T>
@@ -239,7 +90,7 @@ void fft_inplace(Array<T> &in)
     }
 
     cufftHandle plan;
-    find_cufft_plan(plan, rank, t_dims,
+    cufft::findPlan(plan, rank, t_dims,
                     in_embed , istrides[0], istrides[rank],
                     in_embed , istrides[0], istrides[rank],
                     (cufftType)cufft_transform<T>::type, batch);
@@ -274,7 +125,7 @@ Array<Tc> fft_r2c(const Array<Tr> &in)
     dim4 ostrides = out.strides();
 
     cufftHandle plan;
-    find_cufft_plan(plan, rank, t_dims,
+    cufft::findPlan(plan, rank, t_dims,
                     in_embed  , istrides[0], istrides[rank],
                     out_embed , ostrides[0], ostrides[rank],
                     (cufftType)cufft_real_transform<Tc, Tr>::type, batch);
@@ -307,7 +158,7 @@ Array<Tr> fft_c2r(const Array<Tc> &in, const dim4 &odims)
     cufft_real_transform<Tr, Tc> transform;
 
     cufftHandle plan;
-    find_cufft_plan(plan, rank, t_dims,
+    cufft::findPlan(plan, rank, t_dims,
                     in_embed  , istrides[0], istrides[rank],
                     out_embed , ostrides[0], ostrides[rank],
                     (cufftType)cufft_real_transform<Tr, Tc>::type, batch);
diff --git a/src/backend/cuda/kernel/fftconvolve.hpp b/src/backend/cuda/kernel/fftconvolve.hpp
index d7412452a0..399e3f389e 100644
--- a/src/backend/cuda/kernel/fftconvolve.hpp
+++ b/src/backend/cuda/kernel/fftconvolve.hpp
@@ -10,11 +10,9 @@
 #include <backend.hpp>
 #include <dispatch.hpp>
 #include <err_cuda.hpp>
-#include <err_cufft.hpp>
 #include <debug_cuda.hpp>
 #include <Param.hpp>
 #include <memory.hpp>
-#include <cufft.h>
 
 namespace cuda
 {
diff --git a/src/backend/cuda/platform.cpp b/src/backend/cuda/platform.cpp
index 752ea71c96..c696854c14 100644
--- a/src/backend/cuda/platform.cpp
+++ b/src/backend/cuda/platform.cpp
@@ -390,6 +390,11 @@ InteropManager& DeviceManager::getGfxInteropManager()
     return *gfxManager;
 }
 
+cufft::cuFFTPlanner& DeviceManager::getcufftPlanManager()
+{
+    return cufftManagers[cuda::getActiveDeviceId()];
+}
+
 DeviceManager::DeviceManager()
     : cuDevices(0), activeDev(0), nDevices(0), gfxManager(new InteropManager())
 {
diff --git a/src/backend/cuda/platform.hpp b/src/backend/cuda/platform.hpp
index ca7ba418bd..42e16aa969 100644
--- a/src/backend/cuda/platform.hpp
+++ b/src/backend/cuda/platform.hpp
@@ -17,6 +17,8 @@
 #include <fg/window.h>
 #endif
 
+#include <cufftManager.hpp>
+
 namespace cuda
 {
 
@@ -81,6 +83,8 @@ class DeviceManager
 
         InteropManager& getGfxInteropManager();
 
+        cufft::cuFFTPlanner& getcufftPlanManager();
+
         friend std::string getDeviceInfo(int device);
 
         friend std::string getPlatformInfo();
@@ -127,10 +131,13 @@ class DeviceManager
         int activeDev;
         int nDevices;
         cudaStream_t streams[MAX_DEVICES];
+
         //FIXME: Once C++11 has been enabled in CUDA backend
         //shift to use of smart pointer. In this, unique_ptr
         //should be good
         InteropManager* gfxManager;
+
+        cufft::cuFFTPlanner cufftManagers[MAX_DEVICES];
 };
 
 }

From b789dcf0107102d638db0918526e5012b73dcaf7 Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Thu, 22 Dec 2016 19:39:01 +0530
Subject: [PATCH 04/82] Merge cublas handle manager into cuda::DeviceManager

---
 src/backend/cuda/blas.cpp          |  8 ++-
 src/backend/cuda/cublasManager.cpp | 79 ++++++++++++------------------
 src/backend/cuda/cublasManager.hpp | 30 ++++++++++--
 src/backend/cuda/platform.cpp      | 11 +++++
 src/backend/cuda/platform.hpp      |  7 +++
 5 files changed, 78 insertions(+), 57 deletions(-)

diff --git a/src/backend/cuda/blas.cpp b/src/backend/cuda/blas.cpp
index 9d3b9ca7b7..d5c475de6e 100644
--- a/src/backend/cuda/blas.cpp
+++ b/src/backend/cuda/blas.cpp
@@ -25,8 +25,6 @@
 namespace cuda
 {
 
-using cublas::getHandle;
-
 cublasOperation_t
 toCblasTranspose(af_mat_prop opt)
 {
@@ -173,7 +171,7 @@ Array<T> matmul(const Array<T> &lhs, const Array<T> &rhs,
     if(rDims[bColDim] == 1) {
         N = lDims[aColDim];
         CUBLAS_CHECK(gemv_func<T>()(
-                         getHandle(),
+                         DeviceManager::getInstance().getcublasHandle(),
                          lOpts,
                          lDims[0],
                          lDims[1],
@@ -184,7 +182,7 @@ Array<T> matmul(const Array<T> &lhs, const Array<T> &rhs,
                          out.get(), 1));
     } else {
         CUBLAS_CHECK(gemm_func<T>()(
-                         getHandle(),
+                         DeviceManager::getInstance().getcublasHandle(),
                          lOpts,
                          rOpts,
                          M, N, K,
@@ -226,7 +224,7 @@ void trsm(const Array<T> &lhs, Array<T> &rhs, af_mat_prop trans,
     dim4 rStrides = rhs.strides();
 
     CUBLAS_CHECK(trsm_func<T>()(
-                     getHandle(),
+                     DeviceManager::getInstance().getcublasHandle(),
                      is_left  ? CUBLAS_SIDE_LEFT : CUBLAS_SIDE_RIGHT,
                      is_upper ? CUBLAS_FILL_MODE_UPPER : CUBLAS_FILL_MODE_LOWER,
                      toCblasTranspose(trans),
diff --git a/src/backend/cuda/cublasManager.cpp b/src/backend/cuda/cublasManager.cpp
index ca6cfbb2e0..86e167bbef 100644
--- a/src/backend/cuda/cublasManager.cpp
+++ b/src/backend/cuda/cublasManager.cpp
@@ -14,62 +14,43 @@
 #include <boost/scoped_ptr.hpp>
 #include <platform.hpp>
 
-namespace cublas {
+namespace cublas
+{
 
-    const char *errorString(cublasStatus_t err)
-    {
+const char *errorString(cublasStatus_t err)
+{
 
-        switch(err)
-        {
-        case    CUBLAS_STATUS_SUCCESS:              return "CUBLAS_STATUS_SUCCESS";
-        case    CUBLAS_STATUS_NOT_INITIALIZED:      return "CUBLAS_STATUS_NOT_INITIALIZED";
-        case    CUBLAS_STATUS_ALLOC_FAILED:         return "CUBLAS_STATUS_ALLOC_FAILED";
-        case    CUBLAS_STATUS_INVALID_VALUE:        return "CUBLAS_STATUS_INVALID_VALUE";
-        case    CUBLAS_STATUS_ARCH_MISMATCH:        return "CUBLAS_STATUS_ARCH_MISMATCH";
-        case    CUBLAS_STATUS_MAPPING_ERROR:        return "CUBLAS_STATUS_MAPPING_ERROR";
-        case    CUBLAS_STATUS_EXECUTION_FAILED:     return "CUBLAS_STATUS_EXECUTION_FAILED";
-        case    CUBLAS_STATUS_INTERNAL_ERROR:       return "CUBLAS_STATUS_INTERNAL_ERROR";
+    switch(err)
+    {
+    case    CUBLAS_STATUS_SUCCESS:              return "CUBLAS_STATUS_SUCCESS";
+    case    CUBLAS_STATUS_NOT_INITIALIZED:      return "CUBLAS_STATUS_NOT_INITIALIZED";
+    case    CUBLAS_STATUS_ALLOC_FAILED:         return "CUBLAS_STATUS_ALLOC_FAILED";
+    case    CUBLAS_STATUS_INVALID_VALUE:        return "CUBLAS_STATUS_INVALID_VALUE";
+    case    CUBLAS_STATUS_ARCH_MISMATCH:        return "CUBLAS_STATUS_ARCH_MISMATCH";
+    case    CUBLAS_STATUS_MAPPING_ERROR:        return "CUBLAS_STATUS_MAPPING_ERROR";
+    case    CUBLAS_STATUS_EXECUTION_FAILED:     return "CUBLAS_STATUS_EXECUTION_FAILED";
+    case    CUBLAS_STATUS_INTERNAL_ERROR:       return "CUBLAS_STATUS_INTERNAL_ERROR";
 #if CUDA_VERSION > 5050
-        case    CUBLAS_STATUS_NOT_SUPPORTED:        return "CUBLAS_STATUS_NOT_SUPPORTED";
+    case    CUBLAS_STATUS_NOT_SUPPORTED:        return "CUBLAS_STATUS_NOT_SUPPORTED";
 #endif
-        default:                                    return "UNKNOWN";
-        }
+    default:                                    return "UNKNOWN";
     }
+}
 
-    //RAII class around the cublas Handle
-    class cublasHandle
-    {
-        cublasHandle_t handle;
-    public:
-
-        cublasHandle()  : handle(0)
-        {
-            CUBLAS_CHECK(cublasCreate(&handle));
-            CUBLAS_CHECK(cublasSetStream(handle, cuda::getStream(cuda::getActiveDeviceId())));
-        }
-
-        ~cublasHandle()
-        {
-            cublasDestroy(handle);
-        }
-
-        cublasHandle_t get() const
-        {
-            return handle;
-        }
-    };
-
-    cublasHandle_t getHandle()
-    {
-        using boost::scoped_ptr;
-        static scoped_ptr<cublasHandle> handle[cuda::DeviceManager::MAX_DEVICES];
+cublasHandle::cublasHandle()  : handle(0)
+{
+    CUBLAS_CHECK(cublasCreate(&handle));
+    CUBLAS_CHECK(cublasSetStream(handle, cuda::getStream(cuda::getActiveDeviceId())));
+}
 
-        int id = cuda::getActiveDeviceId();
+cublasHandle::~cublasHandle()
+{
+    cublasDestroy(handle);
+}
 
-        if(!handle[id]) {
-            handle[id].reset(new cublasHandle());
-        }
+cublasHandle_t cublasHandle::get() const
+{
+    return handle;
+}
 
-        return handle[id]->get();
-    }
 }
diff --git a/src/backend/cuda/cublasManager.hpp b/src/backend/cuda/cublasManager.hpp
index 52a2793342..41fb567e04 100644
--- a/src/backend/cuda/cublasManager.hpp
+++ b/src/backend/cuda/cublasManager.hpp
@@ -13,11 +13,35 @@
 #include <defines.hpp>
 #include <cublas_v2.h>
 
+namespace cuda
+{
 
-namespace cublas {
+class DeviceManager;
+
+}
+
+namespace cublas
+{
+
+const char * errorString(cublasStatus_t err);
+
+//RAII class around the cublas Handle
+class cublasHandle
+{
+    friend class cuda::DeviceManager;
+
+    public:
+        ~cublasHandle();
+        cublasHandle_t get() const;
+
+    private:
+        cublasHandle();
+        cublasHandle(cublasHandle const&);
+        void operator=(cublasHandle const&);
+
+        cublasHandle_t handle;
+};
 
-    const char * errorString(cublasStatus_t err);
-    cublasHandle_t getHandle();
 }
 
 
diff --git a/src/backend/cuda/platform.cpp b/src/backend/cuda/platform.cpp
index c696854c14..8a571f2df8 100644
--- a/src/backend/cuda/platform.cpp
+++ b/src/backend/cuda/platform.cpp
@@ -395,6 +395,17 @@ cufft::cuFFTPlanner& DeviceManager::getcufftPlanManager()
     return cufftManagers[cuda::getActiveDeviceId()];
 }
 
+cublasHandle_t DeviceManager::getcublasHandle()
+{
+    int id = cuda::getActiveDeviceId();
+
+    if(!cublasHandles[id]) {
+        cublasHandles[id].reset(new cublas::cublasHandle());
+    }
+
+    return cublasHandles[id]->get();
+}
+
 DeviceManager::DeviceManager()
     : cuDevices(0), activeDev(0), nDevices(0), gfxManager(new InteropManager())
 {
diff --git a/src/backend/cuda/platform.hpp b/src/backend/cuda/platform.hpp
index 42e16aa969..c3b1cac8d1 100644
--- a/src/backend/cuda/platform.hpp
+++ b/src/backend/cuda/platform.hpp
@@ -17,7 +17,10 @@
 #include <fg/window.h>
 #endif
 
+#include <boost/scoped_ptr.hpp>
+
 #include <cufftManager.hpp>
+#include <cublasManager.hpp>
 
 namespace cuda
 {
@@ -85,6 +88,8 @@ class DeviceManager
 
         cufft::cuFFTPlanner& getcufftPlanManager();
 
+        cublasHandle_t getcublasHandle();
+
         friend std::string getDeviceInfo(int device);
 
         friend std::string getPlatformInfo();
@@ -138,6 +143,8 @@ class DeviceManager
         InteropManager* gfxManager;
 
         cufft::cuFFTPlanner cufftManagers[MAX_DEVICES];
+
+        boost::scoped_ptr<cublas::cublasHandle> cublasHandles[MAX_DEVICES];
 };
 
 }

From 5e4d401d548eeec83ab67684365c531c910031b0 Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Thu, 22 Dec 2016 20:21:53 +0530
Subject: [PATCH 05/82] Make cuda Manager objects access less hierarchical

---
 src/backend/cuda/blas.cpp          |  6 +++---
 src/backend/cuda/cufftManager.cpp  |  2 +-
 src/backend/cuda/fft.cpp           |  2 +-
 src/backend/cuda/hist_graphics.cpp |  2 +-
 src/backend/cuda/image.cpp         |  2 +-
 src/backend/cuda/platform.cpp      | 24 +++++++++++++++---------
 src/backend/cuda/platform.hpp      | 18 +++++++++++++++---
 src/backend/cuda/plot.cpp          |  2 +-
 src/backend/cuda/surface.cpp       |  2 +-
 src/backend/cuda/vector_field.cpp  |  2 +-
 10 files changed, 40 insertions(+), 22 deletions(-)

diff --git a/src/backend/cuda/blas.cpp b/src/backend/cuda/blas.cpp
index d5c475de6e..3d9d3aba78 100644
--- a/src/backend/cuda/blas.cpp
+++ b/src/backend/cuda/blas.cpp
@@ -171,7 +171,7 @@ Array<T> matmul(const Array<T> &lhs, const Array<T> &rhs,
     if(rDims[bColDim] == 1) {
         N = lDims[aColDim];
         CUBLAS_CHECK(gemv_func<T>()(
-                         DeviceManager::getInstance().getcublasHandle(),
+                         getcublasHandle(),
                          lOpts,
                          lDims[0],
                          lDims[1],
@@ -182,7 +182,7 @@ Array<T> matmul(const Array<T> &lhs, const Array<T> &rhs,
                          out.get(), 1));
     } else {
         CUBLAS_CHECK(gemm_func<T>()(
-                         DeviceManager::getInstance().getcublasHandle(),
+                         getcublasHandle(),
                          lOpts,
                          rOpts,
                          M, N, K,
@@ -224,7 +224,7 @@ void trsm(const Array<T> &lhs, Array<T> &rhs, af_mat_prop trans,
     dim4 rStrides = rhs.strides();
 
     CUBLAS_CHECK(trsm_func<T>()(
-                     DeviceManager::getInstance().getcublasHandle(),
+                     getcublasHandle(),
                      is_left  ? CUBLAS_SIDE_LEFT : CUBLAS_SIDE_RIGHT,
                      is_upper ? CUBLAS_FILL_MODE_UPPER : CUBLAS_FILL_MODE_LOWER,
                      toCblasTranspose(trans),
diff --git a/src/backend/cuda/cufftManager.cpp b/src/backend/cuda/cufftManager.cpp
index f3aea5f45a..28c994b464 100644
--- a/src/backend/cuda/cufftManager.cpp
+++ b/src/backend/cuda/cufftManager.cpp
@@ -114,7 +114,7 @@ void findPlan(cufftHandle &plan, int rank, int *n,
     key_string.append(std::string(key_str_temp));
 
     // find the matching plan_index in the array cuFFTPlanner::mKeys
-    cuFFTPlanner &planner = cuda::DeviceManager::getInstance().getcufftPlanManager();
+    cuFFTPlanner &planner = cuda::getcufftPlanManager();
 
     int planIndex = planner.findIfPlanExists(key_string);
 
diff --git a/src/backend/cuda/fft.cpp b/src/backend/cuda/fft.cpp
index b2b36c90e1..189033e8ae 100644
--- a/src/backend/cuda/fft.cpp
+++ b/src/backend/cuda/fft.cpp
@@ -24,7 +24,7 @@ namespace cuda
 
 void setFFTPlanCacheSize(size_t numPlans)
 {
-    DeviceManager::getInstance().getcufftPlanManager().setMaxCacheSize(numPlans);
+    cuda::getcufftPlanManager().setMaxCacheSize(numPlans);
 }
 
 template<typename T>
diff --git a/src/backend/cuda/hist_graphics.cpp b/src/backend/cuda/hist_graphics.cpp
index d207961f59..750dafae3e 100644
--- a/src/backend/cuda/hist_graphics.cpp
+++ b/src/backend/cuda/hist_graphics.cpp
@@ -25,7 +25,7 @@ void copy_histogram(const Array<T> &data, const forge::Histogram* hist)
     if(DeviceManager::checkGraphicsInteropCapability()) {
         const T *d_P = data.get();
 
-        InteropManager& intrpMngr = DeviceManager::getInstance().getGfxInteropManager();
+        InteropManager& intrpMngr = getGfxInteropManager();
 
         cudaGraphicsResource_t *resources = intrpMngr.getBufferResource(hist);
         // Map resource. Copy data to VBO. Unmap resource.
diff --git a/src/backend/cuda/image.cpp b/src/backend/cuda/image.cpp
index ca87d3129f..0492c9c6d4 100644
--- a/src/backend/cuda/image.cpp
+++ b/src/backend/cuda/image.cpp
@@ -28,7 +28,7 @@ template<typename T>
 void copy_image(const Array<T> &in, const forge::Image* image)
 {
     if(DeviceManager::checkGraphicsInteropCapability()) {
-        InteropManager& intrpMngr = DeviceManager::getInstance().getGfxInteropManager();
+        InteropManager& intrpMngr = getGfxInteropManager();
 
         cudaGraphicsResource_t *resources = intrpMngr.getBufferResource(image);
 
diff --git a/src/backend/cuda/platform.cpp b/src/backend/cuda/platform.cpp
index 8a571f2df8..79ec7e021f 100644
--- a/src/backend/cuda/platform.cpp
+++ b/src/backend/cuda/platform.cpp
@@ -385,25 +385,26 @@ DeviceManager::~DeviceManager()
     if (gfxManager) delete gfxManager;
 }
 
-InteropManager& DeviceManager::getGfxInteropManager()
+InteropManager& getGfxInteropManager()
 {
-    return *gfxManager;
+    return *(DeviceManager::getInstance().gfxManager);
 }
 
-cufft::cuFFTPlanner& DeviceManager::getcufftPlanManager()
+cufft::cuFFTPlanner& getcufftPlanManager()
 {
-    return cufftManagers[cuda::getActiveDeviceId()];
+    return DeviceManager::getInstance().cufftManagers[cuda::getActiveDeviceId()];
 }
 
-cublasHandle_t DeviceManager::getcublasHandle()
+cublasHandle_t getcublasHandle()
 {
+    DeviceManager& instance = DeviceManager::getInstance();
+
     int id = cuda::getActiveDeviceId();
 
-    if(!cublasHandles[id]) {
-        cublasHandles[id].reset(new cublas::cublasHandle());
-    }
+    if (!(instance.cublasHandles[id]))
+        instance.resetcublasHandle(id);
 
-    return cublasHandles[id]->get();
+    return instance.cublasHandles[id]->get();
 }
 
 DeviceManager::DeviceManager()
@@ -528,6 +529,11 @@ int DeviceManager::setActiveDevice(int device, int nId)
     return old;
 }
 
+void DeviceManager::resetcublasHandle(int device)
+{
+    cublasHandles[device].reset(new cublas::cublasHandle());
+}
+
 void sync(int device)
 {
     int currDevice = getActiveDeviceId();
diff --git a/src/backend/cuda/platform.hpp b/src/backend/cuda/platform.hpp
index c3b1cac8d1..163df71f03 100644
--- a/src/backend/cuda/platform.hpp
+++ b/src/backend/cuda/platform.hpp
@@ -73,6 +73,16 @@ bool& evalFlag();
 
 class InteropManager;
 
+///////////////////////// BEGIN Sub-Managers ///////////////////
+//
+InteropManager& getGfxInteropManager();
+
+cufft::cuFFTPlanner& getcufftPlanManager();
+
+cublasHandle_t getcublasHandle();
+//
+///////////////////////// END Sub-Managers /////////////////////
+
 class DeviceManager
 {
     public:
@@ -84,11 +94,11 @@ class DeviceManager
 
         ~DeviceManager();
 
-        InteropManager& getGfxInteropManager();
+        friend InteropManager& getGfxInteropManager();
 
-        cufft::cuFFTPlanner& getcufftPlanManager();
+        friend cufft::cuFFTPlanner& getcufftPlanManager();
 
-        cublasHandle_t getcublasHandle();
+        friend cublasHandle_t getcublasHandle();
 
         friend std::string getDeviceInfo(int device);
 
@@ -133,6 +143,8 @@ class DeviceManager
 
         int setActiveDevice(int device, int native = -1);
 
+        void resetcublasHandle(int device);
+
         int activeDev;
         int nDevices;
         cudaStream_t streams[MAX_DEVICES];
diff --git a/src/backend/cuda/plot.cpp b/src/backend/cuda/plot.cpp
index 40ae7b4574..437c276226 100644
--- a/src/backend/cuda/plot.cpp
+++ b/src/backend/cuda/plot.cpp
@@ -30,7 +30,7 @@ void copy_plot(const Array<T> &P, forge::Plot* plot)
     if(DeviceManager::checkGraphicsInteropCapability()) {
         const T *d_P = P.get();
 
-        InteropManager& intrpMngr = DeviceManager::getInstance().getGfxInteropManager();
+        InteropManager& intrpMngr = getGfxInteropManager();
 
         cudaGraphicsResource_t *resources = intrpMngr.getBufferResource(plot);
         // Map resource. Copy data to VBO. Unmap resource.
diff --git a/src/backend/cuda/surface.cpp b/src/backend/cuda/surface.cpp
index 18385decd6..749ff859d6 100644
--- a/src/backend/cuda/surface.cpp
+++ b/src/backend/cuda/surface.cpp
@@ -30,7 +30,7 @@ void copy_surface(const Array<T> &P, forge::Surface* surface)
     if(DeviceManager::checkGraphicsInteropCapability()) {
         const T *d_P = P.get();
 
-        InteropManager& intrpMngr = DeviceManager::getInstance().getGfxInteropManager();
+        InteropManager& intrpMngr = getGfxInteropManager();
 
         cudaGraphicsResource_t *resources = intrpMngr.getBufferResource(surface);
         // Map resource. Copy data to VBO. Unmap resource.
diff --git a/src/backend/cuda/vector_field.cpp b/src/backend/cuda/vector_field.cpp
index 6fa4dd9fee..aab3f13036 100644
--- a/src/backend/cuda/vector_field.cpp
+++ b/src/backend/cuda/vector_field.cpp
@@ -26,7 +26,7 @@ void copy_vector_field(const Array<T> &points, const Array<T> &directions,
                        forge::VectorField* vector_field)
 {
     if(DeviceManager::checkGraphicsInteropCapability()) {
-        InteropManager& intrpMngr = DeviceManager::getInstance().getGfxInteropManager();
+        InteropManager& intrpMngr = getGfxInteropManager();
 
         cudaGraphicsResource_t *resources = intrpMngr.getBufferResource(vector_field);
 

From b75419476ffb31199e84d732b5c5ce226d074181 Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Thu, 22 Dec 2016 20:36:36 +0530
Subject: [PATCH 06/82] Wrap interopManager pointer with boost::scoped_ptr

---
 src/backend/cuda/platform.cpp | 7 +------
 src/backend/cuda/platform.hpp | 7 +------
 2 files changed, 2 insertions(+), 12 deletions(-)

diff --git a/src/backend/cuda/platform.cpp b/src/backend/cuda/platform.cpp
index 79ec7e021f..0b1123df2b 100644
--- a/src/backend/cuda/platform.cpp
+++ b/src/backend/cuda/platform.cpp
@@ -380,14 +380,9 @@ DeviceManager& DeviceManager::getInstance()
     return my_instance;
 }
 
-DeviceManager::~DeviceManager()
-{
-    if (gfxManager) delete gfxManager;
-}
-
 InteropManager& getGfxInteropManager()
 {
-    return *(DeviceManager::getInstance().gfxManager);
+    return *(DeviceManager::getInstance().gfxManager.get());
 }
 
 cufft::cuFFTPlanner& getcufftPlanManager()
diff --git a/src/backend/cuda/platform.hpp b/src/backend/cuda/platform.hpp
index 163df71f03..9b15457fda 100644
--- a/src/backend/cuda/platform.hpp
+++ b/src/backend/cuda/platform.hpp
@@ -92,8 +92,6 @@ class DeviceManager
 
         static DeviceManager& getInstance();
 
-        ~DeviceManager();
-
         friend InteropManager& getGfxInteropManager();
 
         friend cufft::cuFFTPlanner& getcufftPlanManager();
@@ -149,10 +147,7 @@ class DeviceManager
         int nDevices;
         cudaStream_t streams[MAX_DEVICES];
 
-        //FIXME: Once C++11 has been enabled in CUDA backend
-        //shift to use of smart pointer. In this, unique_ptr
-        //should be good
-        InteropManager* gfxManager;
+        boost::scoped_ptr<InteropManager> gfxManager;
 
         cufft::cuFFTPlanner cufftManagers[MAX_DEVICES];
 

From de3f9f747934beebe0a7ddabc6a2b952ff018b04 Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Thu, 22 Dec 2016 21:13:07 +0530
Subject: [PATCH 07/82] Merge cusolver handle manager into cuda::DeviceManager

---
 src/backend/cuda/cholesky.cu           |   8 +-
 src/backend/cuda/cusolverDnManager.cpp | 101 ++++++++-----------------
 src/backend/cuda/cusolverDnManager.hpp |  56 ++++++++++----
 src/backend/cuda/lu.cu                 |   8 +-
 src/backend/cuda/platform.cpp          |  32 ++++++++
 src/backend/cuda/platform.hpp          |   9 +++
 src/backend/cuda/qr.cu                 |  14 ++--
 src/backend/cuda/solve.cu              |  20 +++--
 src/backend/cuda/svd.cu                |   8 +-
 9 files changed, 138 insertions(+), 118 deletions(-)

diff --git a/src/backend/cuda/cholesky.cu b/src/backend/cuda/cholesky.cu
index c6869dc6a6..db90dba238 100644
--- a/src/backend/cuda/cholesky.cu
+++ b/src/backend/cuda/cholesky.cu
@@ -12,7 +12,7 @@
 
 #if defined(WITH_CUDA_LINEAR_ALGEBRA)
 
-#include <cusolverDnManager.hpp>
+#include <platform.hpp>
 #include <cublas_v2.h>
 #include <identity.hpp>
 #include <iostream>
@@ -26,8 +26,6 @@
 namespace cuda
 {
 
-using cusolver::getDnHandle;
-
 //cusolverStatus_t cusolverDn<>potrf_bufferSize(
 //        cusolverDnHandle_t handle,
 //        cublasFillMode_t uplo,
@@ -114,7 +112,7 @@ int cholesky_inplace(Array<T> &in, const bool is_upper)
     if(is_upper)
         uplo = CUBLAS_FILL_MODE_UPPER;
 
-    CUSOLVER_CHECK(potrf_buf_func<T>()(getDnHandle(),
+    CUSOLVER_CHECK(potrf_buf_func<T>()(getcusolverDnHandle(),
                                        uplo,
                                        N,
                                        in.get(), in.strides()[1],
@@ -123,7 +121,7 @@ int cholesky_inplace(Array<T> &in, const bool is_upper)
     T *workspace = memAlloc<T>(lwork);
     int *d_info = memAlloc<int>(1);
 
-    CUSOLVER_CHECK(potrf_func<T>()(getDnHandle(),
+    CUSOLVER_CHECK(potrf_func<T>()(getcusolverDnHandle(),
                                    uplo,
                                    N,
                                    in.get(), in.strides()[1],
diff --git a/src/backend/cuda/cusolverDnManager.cpp b/src/backend/cuda/cusolverDnManager.cpp
index 3fa1f9ce11..3dc9aa0b2d 100644
--- a/src/backend/cuda/cusolverDnManager.cpp
+++ b/src/backend/cuda/cusolverDnManager.cpp
@@ -18,79 +18,44 @@
 #include <iostream>
 #include <boost/scoped_ptr.hpp>
 
-namespace cusolver {
-
-    const char *errorString(cusolverStatus_t err)
-    {
-        switch(err) {
-        case    CUSOLVER_STATUS_SUCCESS                     :   return "CUSOLVER_STATUS_SUCCESS"                    ;
-        case    CUSOLVER_STATUS_NOT_INITIALIZED             :   return "CUSOLVER_STATUS_NOT_INITIALIZED"            ;
-        case    CUSOLVER_STATUS_ALLOC_FAILED                :   return "CUSOLVER_STATUS_ALLOC_FAILED"               ;
-        case    CUSOLVER_STATUS_INVALID_VALUE               :   return "CUSOLVER_STATUS_INVALID_VALUE"              ;
-        case    CUSOLVER_STATUS_ARCH_MISMATCH               :   return "CUSOLVER_STATUS_ARCH_MISMATCH"              ;
-        case    CUSOLVER_STATUS_MAPPING_ERROR               :   return "CUSOLVER_STATUS_MAPPING_ERROR"              ;
-        case    CUSOLVER_STATUS_EXECUTION_FAILED            :   return "CUSOLVER_STATUS_EXECUTION_FAILED"           ;
-        case    CUSOLVER_STATUS_INTERNAL_ERROR              :   return "CUSOLVER_STATUS_INTERNAL_ERROR"             ;
-        case    CUSOLVER_STATUS_MATRIX_TYPE_NOT_SUPPORTED   :   return "CUSOLVER_STATUS_MATRIX_TYPE_NOT_SUPPORTED"  ;
-        case    CUSOLVER_STATUS_NOT_SUPPORTED               :   return "CUSOLVER_STATUS_NOT_SUPPORTED"              ;
-        case    CUSOLVER_STATUS_ZERO_PIVOT                  :   return "CUSOLVER_STATUS_ZERO_PIVOT"                 ;
-        case    CUSOLVER_STATUS_INVALID_LICENSE             :   return "CUSOLVER_STATUS_INVALID_LICENSE"            ;
-        default                                             :   return "UNKNOWN";
-        }
+namespace cusolver
+{
+
+const char *errorString(cusolverStatus_t err)
+{
+    switch(err) {
+    case    CUSOLVER_STATUS_SUCCESS                     :   return "CUSOLVER_STATUS_SUCCESS"                    ;
+    case    CUSOLVER_STATUS_NOT_INITIALIZED             :   return "CUSOLVER_STATUS_NOT_INITIALIZED"            ;
+    case    CUSOLVER_STATUS_ALLOC_FAILED                :   return "CUSOLVER_STATUS_ALLOC_FAILED"               ;
+    case    CUSOLVER_STATUS_INVALID_VALUE               :   return "CUSOLVER_STATUS_INVALID_VALUE"              ;
+    case    CUSOLVER_STATUS_ARCH_MISMATCH               :   return "CUSOLVER_STATUS_ARCH_MISMATCH"              ;
+    case    CUSOLVER_STATUS_MAPPING_ERROR               :   return "CUSOLVER_STATUS_MAPPING_ERROR"              ;
+    case    CUSOLVER_STATUS_EXECUTION_FAILED            :   return "CUSOLVER_STATUS_EXECUTION_FAILED"           ;
+    case    CUSOLVER_STATUS_INTERNAL_ERROR              :   return "CUSOLVER_STATUS_INTERNAL_ERROR"             ;
+    case    CUSOLVER_STATUS_MATRIX_TYPE_NOT_SUPPORTED   :   return "CUSOLVER_STATUS_MATRIX_TYPE_NOT_SUPPORTED"  ;
+    case    CUSOLVER_STATUS_NOT_SUPPORTED               :   return "CUSOLVER_STATUS_NOT_SUPPORTED"              ;
+    case    CUSOLVER_STATUS_ZERO_PIVOT                  :   return "CUSOLVER_STATUS_ZERO_PIVOT"                 ;
+    case    CUSOLVER_STATUS_INVALID_LICENSE             :   return "CUSOLVER_STATUS_INVALID_LICENSE"            ;
+    default                                             :   return "UNKNOWN";
     }
+}
 
 
-//RAII class around the cusolver Handle
-    class cusolverDnHandle
-    {
-        cusolverDnHandle_t handle;
-    public:
-
-        cusolverDnHandle()
-            : handle(0)
-        {
-            CUSOLVER_CHECK(cusolverDnCreate(&handle));
-        }
-
-        ~cusolverDnHandle()
-        {
-            cusolverDnDestroy(handle);
-        }
-
-        cusolverDnHandle_t get() const
-        {
-            return handle;
-        }
-    };
-
-    cusolverDnHandle_t getDnHandle()
-    {
-        using boost::scoped_ptr;
-        static scoped_ptr<cusolverDnHandle> handle[cuda::DeviceManager::MAX_DEVICES];
-
-        int id = cuda::getActiveDeviceId();
-
-        if(!handle[id]) {
-            handle[id].reset(new cusolverDnHandle());
-        }
+cusolverDnHandle::cusolverDnHandle()
+    : handle(0)
+{
+    CUSOLVER_CHECK(cusolverDnCreate(&handle));
+}
 
-        // FIXME
-        // This is not an ideal case. It's just a hack.
-        // The correct way to do is to use
-        // CUSOLVER_CHECK(cusolverDnSetStream(cuda::getStream(cuda::getActiveDeviceId())))
-        // in the class constructor.
-        // However, this is causing a lot of the cusolver functions to fail.
-        // The only way to fix them is to use cudaDeviceSynchronize() and cudaStreamSynchronize()
-        // all over the place, but even then some calls like getrs in solve_lu
-        // continue to fail on any stream other than 0.
-        //
-        // cuSolver Streams patch:
-        // https://gist.github.com/shehzan10/414c3d04a40e7c4a03ed3c2e1b9072e7
-        //
-        CUDA_CHECK(cudaStreamSynchronize(cuda::getStream(id)));
+cusolverDnHandle::~cusolverDnHandle()
+{
+    cusolverDnDestroy(handle);
+}
 
-        return handle[id]->get();
-    }
+cusolverDnHandle_t cusolverDnHandle::get() const
+{
+    return handle;
+}
 
 }
 
diff --git a/src/backend/cuda/cusolverDnManager.hpp b/src/backend/cuda/cusolverDnManager.hpp
index dbaa694d58..0793297efb 100644
--- a/src/backend/cuda/cusolverDnManager.hpp
+++ b/src/backend/cuda/cusolverDnManager.hpp
@@ -16,27 +16,51 @@
 #include <defines.hpp>
 #include <err_common.hpp>
 
+namespace cuda
+{
+
+class DeviceManager;
+
+}
+
 namespace cusolver
 {
 
-    const char * errorString(cusolverStatus_t err);
-    cusolverDnHandle_t getDnHandle();
+const char * errorString(cusolverStatus_t err);
+
+//RAII class around the cusolver Handle
+class cusolverDnHandle
+{
+    friend class cuda::DeviceManager;
+
+    public:
+        ~cusolverDnHandle();
+        cusolverDnHandle_t get() const;
+
+    private:
+        cusolverDnHandle();
+        cusolverDnHandle(cusolverDnHandle const&);
+        void operator=(cusolverDnHandle const&);
+
+        cusolverDnHandle_t handle;
+};
+
 }
 
 #define CUSOLVER_CHECK(fn) do {                     \
-        cusolverStatus_t _error = fn;               \
-        if (_error != CUSOLVER_STATUS_SUCCESS) {    \
-            char _err_msg[1024];                    \
-            snprintf(_err_msg,                      \
-                     sizeof(_err_msg),              \
-                     "CUBLAS Error (%d): %s\n",     \
-                     (int)(_error),                 \
-                     cusolver::errorString(         \
-                         _error));                  \
-                                                    \
-            AF_ERROR(_err_msg,                      \
-                     AF_ERR_INTERNAL);              \
-        }                                           \
-    } while(0)
+    cusolverStatus_t _error = fn;               \
+    if (_error != CUSOLVER_STATUS_SUCCESS) {    \
+        char _err_msg[1024];                    \
+        snprintf(_err_msg,                      \
+                sizeof(_err_msg),              \
+                "CUBLAS Error (%d): %s\n",     \
+                (int)(_error),                 \
+                cusolver::errorString(         \
+                    _error));                  \
+        \
+        AF_ERROR(_err_msg,                      \
+                AF_ERR_INTERNAL);              \
+    }                                           \
+} while(0)
 
 #endif
diff --git a/src/backend/cuda/lu.cu b/src/backend/cuda/lu.cu
index ce0b545a84..b4a7b946b2 100644
--- a/src/backend/cuda/lu.cu
+++ b/src/backend/cuda/lu.cu
@@ -12,7 +12,7 @@
 
 #if defined(WITH_CUDA_LINEAR_ALGEBRA)
 
-#include <cusolverDnManager.hpp>
+#include <platform.hpp>
 #include <memory.hpp>
 #include <copy.hpp>
 #include <math.hpp>
@@ -23,8 +23,6 @@
 namespace cuda
 {
 
-using cusolver::getDnHandle;
-
 //cusolverStatus_t CUDENSEAPI cusolverDn<>getrf_bufferSize(
 //        cusolverDnHandle_t handle,
 //        int m, int n,
@@ -133,7 +131,7 @@ Array<int> lu_inplace(Array<T> &in, const bool convert_pivot)
 
     int lwork = 0;
 
-    CUSOLVER_CHECK(getrf_buf_func<T>()(getDnHandle(),
+    CUSOLVER_CHECK(getrf_buf_func<T>()(getcusolverDnHandle(),
                                        M, N,
                                        in.get(), in.strides()[1],
                                        &lwork));
@@ -141,7 +139,7 @@ Array<int> lu_inplace(Array<T> &in, const bool convert_pivot)
     T *workspace = memAlloc<T>(lwork);
     int *info = memAlloc<int>(1);
 
-    CUSOLVER_CHECK(getrf_func<T>()(getDnHandle(),
+    CUSOLVER_CHECK(getrf_func<T>()(getcusolverDnHandle(),
                                    M, N,
                                    in.get(), in.strides()[1],
                                    workspace,
diff --git a/src/backend/cuda/platform.cpp b/src/backend/cuda/platform.cpp
index 0b1123df2b..4e6d581e12 100644
--- a/src/backend/cuda/platform.cpp
+++ b/src/backend/cuda/platform.cpp
@@ -402,6 +402,33 @@ cublasHandle_t getcublasHandle()
     return instance.cublasHandles[id]->get();
 }
 
+cusolverDnHandle_t getcusolverDnHandle()
+{
+    DeviceManager& instance = DeviceManager::getInstance();
+
+    int id = cuda::getActiveDeviceId();
+
+    if (!(instance.cusolverHandles[id]))
+        instance.resetcusolverHandle(id);
+
+    // FIXME
+    // This is not an ideal case. It's just a hack.
+    // The correct way to do is to use
+    // CUSOLVER_CHECK(cusolverDnSetStream(cuda::getStream(cuda::getActiveDeviceId())))
+    // in the class constructor.
+    // However, this is causing a lot of the cusolver functions to fail.
+    // The only way to fix them is to use cudaDeviceSynchronize() and cudaStreamSynchronize()
+    // all over the place, but even then some calls like getrs in solve_lu
+    // continue to fail on any stream other than 0.
+    //
+    // cuSolver Streams patch:
+    // https://gist.github.com/shehzan10/414c3d04a40e7c4a03ed3c2e1b9072e7
+    //
+    CUDA_CHECK(cudaStreamSynchronize(cuda::getStream(id)));
+
+    return instance.cusolverHandles[id]->get();
+}
+
 DeviceManager::DeviceManager()
     : cuDevices(0), activeDev(0), nDevices(0), gfxManager(new InteropManager())
 {
@@ -529,6 +556,11 @@ void DeviceManager::resetcublasHandle(int device)
     cublasHandles[device].reset(new cublas::cublasHandle());
 }
 
+void DeviceManager::resetcusolverHandle(int device)
+{
+    cusolverHandles[device].reset(new cusolver::cusolverDnHandle());
+}
+
 void sync(int device)
 {
     int currDevice = getActiveDeviceId();
diff --git a/src/backend/cuda/platform.hpp b/src/backend/cuda/platform.hpp
index 9b15457fda..e87a843f0b 100644
--- a/src/backend/cuda/platform.hpp
+++ b/src/backend/cuda/platform.hpp
@@ -21,6 +21,7 @@
 
 #include <cufftManager.hpp>
 #include <cublasManager.hpp>
+#include <cusolverDnManager.hpp>
 
 namespace cuda
 {
@@ -80,6 +81,8 @@ InteropManager& getGfxInteropManager();
 cufft::cuFFTPlanner& getcufftPlanManager();
 
 cublasHandle_t getcublasHandle();
+
+cusolverDnHandle_t getcusolverDnHandle();
 //
 ///////////////////////// END Sub-Managers /////////////////////
 
@@ -98,6 +101,8 @@ class DeviceManager
 
         friend cublasHandle_t getcublasHandle();
 
+        friend cusolverDnHandle_t getcusolverDnHandle();
+
         friend std::string getDeviceInfo(int device);
 
         friend std::string getPlatformInfo();
@@ -143,6 +148,8 @@ class DeviceManager
 
         void resetcublasHandle(int device);
 
+        void resetcusolverHandle(int device);
+
         int activeDev;
         int nDevices;
         cudaStream_t streams[MAX_DEVICES];
@@ -152,6 +159,8 @@ class DeviceManager
         cufft::cuFFTPlanner cufftManagers[MAX_DEVICES];
 
         boost::scoped_ptr<cublas::cublasHandle> cublasHandles[MAX_DEVICES];
+
+        boost::scoped_ptr<cusolver::cusolverDnHandle> cusolverHandles[MAX_DEVICES];
 };
 
 }
diff --git a/src/backend/cuda/qr.cu b/src/backend/cuda/qr.cu
index 41ad1c2600..d3dc20374f 100644
--- a/src/backend/cuda/qr.cu
+++ b/src/backend/cuda/qr.cu
@@ -12,7 +12,7 @@
 
 #if defined(WITH_CUDA_LINEAR_ALGEBRA)
 
-#include <cusolverDnManager.hpp>
+#include <platform.hpp>
 #include <cublas_v2.h>
 #include <identity.hpp>
 #include <memory.hpp>
@@ -26,8 +26,6 @@
 namespace cuda
 {
 
-using cusolver::getDnHandle;
-
 //cusolverStatus_t cusolverDn<>geqrf_bufferSize(
 //        cusolverDnHandle_t handle,
 //        int m, int n,
@@ -134,7 +132,7 @@ void qr(Array<T> &q, Array<T> &r, Array<T> &t, const Array<T> &in)
 
     int lwork = 0;
 
-    CUSOLVER_CHECK(geqrf_buf_func<T>()(getDnHandle(),
+    CUSOLVER_CHECK(geqrf_buf_func<T>()(getcusolverDnHandle(),
                                        M, N,
                                        in_copy.get(), in_copy.strides()[1],
                                        &lwork));
@@ -144,7 +142,7 @@ void qr(Array<T> &q, Array<T> &r, Array<T> &t, const Array<T> &in)
     t = createEmptyArray<T>(af::dim4(min(M, N), 1, 1, 1));
     int *info = memAlloc<int>(1);
 
-    CUSOLVER_CHECK(geqrf_func<T>()(getDnHandle(),
+    CUSOLVER_CHECK(geqrf_func<T>()(getcusolverDnHandle(),
                                    M, N,
                                    in_copy.get(), in_copy.strides()[1],
                                    t.get(),
@@ -161,7 +159,7 @@ void qr(Array<T> &q, Array<T> &r, Array<T> &t, const Array<T> &in)
     dim4 qdims(M, mn);
     q = identity<T>(qdims);
 
-    CUSOLVER_CHECK(mqr_func<T>()(getDnHandle(),
+    CUSOLVER_CHECK(mqr_func<T>()(getcusolverDnHandle(),
                                  CUBLAS_SIDE_LEFT, CUBLAS_OP_N,
                                  q.dims()[0],
                                  q.dims()[1],
@@ -189,7 +187,7 @@ Array<T> qr_inplace(Array<T> &in)
 
     int lwork = 0;
 
-    CUSOLVER_CHECK(geqrf_buf_func<T>()(getDnHandle(),
+    CUSOLVER_CHECK(geqrf_buf_func<T>()(getcusolverDnHandle(),
                                        M, N,
                                        in.get(), in.strides()[1],
                                        &lwork));
@@ -197,7 +195,7 @@ Array<T> qr_inplace(Array<T> &in)
     T *workspace = memAlloc<T>(lwork);
     int *info = memAlloc<int>(1);
 
-    CUSOLVER_CHECK(geqrf_func<T>()(getDnHandle(),
+    CUSOLVER_CHECK(geqrf_func<T>()(getcusolverDnHandle(),
                                    M, N,
                                    in.get(), in.strides()[1],
                                    t.get(),
diff --git a/src/backend/cuda/solve.cu b/src/backend/cuda/solve.cu
index 9db4889e2f..8d42a53227 100644
--- a/src/backend/cuda/solve.cu
+++ b/src/backend/cuda/solve.cu
@@ -12,7 +12,7 @@
 
 #if defined(WITH_CUDA_LINEAR_ALGEBRA)
 
-#include <cusolverDnManager.hpp>
+#include <platform.hpp>
 #include <cublas_v2.h>
 #include <identity.hpp>
 #include <iostream>
@@ -32,8 +32,6 @@
 namespace cuda
 {
 
-using cusolver::getDnHandle;
-
 //cusolverStatus_t cusolverDn<>getrs(
 //    cusolverDnHandle_t handle,
 //    cublasOperation_t trans,
@@ -178,7 +176,7 @@ Array<T> solveLU(const Array<T> &A, const Array<int> &pivot,
 
     int *info = memAlloc<int>(1);
 
-    CUSOLVER_CHECK(getrs_func<T>()(getDnHandle(),
+    CUSOLVER_CHECK(getrs_func<T>()(getcusolverDnHandle(),
                                    CUBLAS_OP_N,
                                    N, NRHS,
                                    A.get(), A.strides()[1],
@@ -203,7 +201,7 @@ Array<T> generalSolve(const Array<T> &a, const Array<T> &b)
 
     int *info = memAlloc<int>(1);
 
-    CUSOLVER_CHECK(getrs_func<T>()(getDnHandle(),
+    CUSOLVER_CHECK(getrs_func<T>()(getcusolverDnHandle(),
                                    CUBLAS_OP_N,
                                    N, K,
                                    A.get(), A.strides()[1],
@@ -246,7 +244,7 @@ Array<T> leastSquares(const Array<T> &a, const Array<T> &b)
         int lwork = 0;
 
         // Get workspace needed for QR
-        CUSOLVER_CHECK(geqrf_solve_buf_func<T>()(getDnHandle(),
+        CUSOLVER_CHECK(geqrf_solve_buf_func<T>()(getcusolverDnHandle(),
                                                  A.dims()[0], A.dims()[1],
                                                  A.get(), A.strides()[1],
                                                  &lwork));
@@ -256,7 +254,7 @@ Array<T> leastSquares(const Array<T> &a, const Array<T> &b)
         int *info = memAlloc<int>(1);
 
         // In place Perform in place QR
-        CUSOLVER_CHECK(geqrf_solve_func<T>()(getDnHandle(),
+        CUSOLVER_CHECK(geqrf_solve_func<T>()(getcusolverDnHandle(),
                                              A.dims()[0], A.dims()[1],
                                              A.get(), A.strides()[1],
                                              t.get(),
@@ -274,7 +272,7 @@ Array<T> leastSquares(const Array<T> &a, const Array<T> &b)
         B.resetDims(dim4(N, K));
 
         // matmul(Q, Bpad)
-        CUSOLVER_CHECK(mqr_solve_func<T>()(getDnHandle(),
+        CUSOLVER_CHECK(mqr_solve_func<T>()(getcusolverDnHandle(),
                                            CUBLAS_SIDE_LEFT, CUBLAS_OP_N,
                                            B.dims()[0],
                                            B.dims()[1],
@@ -304,7 +302,7 @@ Array<T> leastSquares(const Array<T> &a, const Array<T> &b)
         int lwork = 0;
 
         // Get workspace needed for QR
-        CUSOLVER_CHECK(geqrf_solve_buf_func<T>()(getDnHandle(),
+        CUSOLVER_CHECK(geqrf_solve_buf_func<T>()(getcusolverDnHandle(),
                                                  A.dims()[0], A.dims()[1],
                                                  A.get(), A.strides()[1],
                                                  &lwork));
@@ -314,7 +312,7 @@ Array<T> leastSquares(const Array<T> &a, const Array<T> &b)
         int *info = memAlloc<int>(1);
 
         // In place Perform in place QR
-        CUSOLVER_CHECK(geqrf_solve_func<T>()(getDnHandle(),
+        CUSOLVER_CHECK(geqrf_solve_func<T>()(getcusolverDnHandle(),
                                              A.dims()[0], A.dims()[1],
                                              A.get(), A.strides()[1],
                                              t.get(),
@@ -322,7 +320,7 @@ Array<T> leastSquares(const Array<T> &a, const Array<T> &b)
                                              info));
 
         // matmul(Q1, B)
-        CUSOLVER_CHECK(mqr_solve_func<T>()(getDnHandle(),
+        CUSOLVER_CHECK(mqr_solve_func<T>()(getcusolverDnHandle(),
                                            CUBLAS_SIDE_LEFT,
                                            trans<T>(),
                                            M, K, N,
diff --git a/src/backend/cuda/svd.cu b/src/backend/cuda/svd.cu
index e07c1f0564..5c9d58f74e 100644
--- a/src/backend/cuda/svd.cu
+++ b/src/backend/cuda/svd.cu
@@ -10,7 +10,7 @@
 #include <svd.hpp>
 #include <err_common.hpp>
 
-#include <cusolverDnManager.hpp>
+#include <platform.hpp>
 #include "transpose.hpp"
 #include <memory.hpp>
 #include <copy.hpp>
@@ -23,8 +23,6 @@
 
 namespace cuda
 {
-    using cusolver::getDnHandle;
-
     template<typename T>
     cusolverStatus_t gesvd_buf_func(cusolverDnHandle_t handle, int m, int n, int *Lwork)
     {
@@ -90,14 +88,14 @@ SVD_SPECIALIZE(cdouble, double, Z);
 
         int lwork = 0;
 
-        CUSOLVER_CHECK(gesvd_buf_func<T>(getDnHandle(), M, N, &lwork));
+        CUSOLVER_CHECK(gesvd_buf_func<T>(getcusolverDnHandle(), M, N, &lwork));
 
         T  *lWorkspace = memAlloc<T >(lwork);
         Tr *rWorkspace = memAlloc<Tr>(5 * std::min(M, N));
 
         int *info = memAlloc<int>(1);
 
-        gesvd_func<T, Tr>(getDnHandle(), 'A', 'A', M, N, in.get(),
+        gesvd_func<T, Tr>(getcusolverDnHandle(), 'A', 'A', M, N, in.get(),
                           M, s.get(), u.get(), M, vt.get(), N,
                           lWorkspace, lwork, rWorkspace, info);
 

From 5689b849197c64f355184ec3afa35050714fb84e Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Fri, 23 Dec 2016 06:20:00 +0530
Subject: [PATCH 08/82] Merge cusparse handle manager into cuda::DeviceManager

---
 src/backend/cuda/cusparseManager.cpp | 85 +++++++++++-----------------
 src/backend/cuda/cusparseManager.hpp | 31 +++++++++-
 src/backend/cuda/platform.cpp        | 17 ++++++
 src/backend/cuda/platform.hpp        |  9 +++
 src/backend/cuda/sparse.cu           | 32 +++++------
 src/backend/cuda/sparse_blas.cpp     |  6 +-
 6 files changed, 103 insertions(+), 77 deletions(-)

diff --git a/src/backend/cuda/cusparseManager.cpp b/src/backend/cuda/cusparseManager.cpp
index 24f0c40045..52fc40e4bc 100644
--- a/src/backend/cuda/cusparseManager.cpp
+++ b/src/backend/cuda/cusparseManager.cpp
@@ -13,64 +13,43 @@
 #include <stdexcept>
 #include <string>
 #include <iostream>
-#include <boost/scoped_ptr.hpp>
 
-namespace cusparse {
-
-    const char *errorString(cusparseStatus_t err)
-    {
-        switch(err) {
-        case    CUSPARSE_STATUS_SUCCESS                     :   return "CUSPARSE_STATUS_SUCCESS"                    ;
-        case    CUSPARSE_STATUS_NOT_INITIALIZED             :   return "CUSPARSE_STATUS_NOT_INITIALIZED"            ;
-        case    CUSPARSE_STATUS_ALLOC_FAILED                :   return "CUSPARSE_STATUS_ALLOC_FAILED"               ;
-        case    CUSPARSE_STATUS_INVALID_VALUE               :   return "CUSPARSE_STATUS_INVALID_VALUE"              ;
-        case    CUSPARSE_STATUS_ARCH_MISMATCH               :   return "CUSPARSE_STATUS_ARCH_MISMATCH"              ;
-        case    CUSPARSE_STATUS_MAPPING_ERROR               :   return "CUSPARSE_STATUS_MAPPING_ERROR"              ;
-        case    CUSPARSE_STATUS_EXECUTION_FAILED            :   return "CUSPARSE_STATUS_EXECUTION_FAILED"           ;
-        case    CUSPARSE_STATUS_INTERNAL_ERROR              :   return "CUSPARSE_STATUS_INTERNAL_ERROR"             ;
-        case    CUSPARSE_STATUS_MATRIX_TYPE_NOT_SUPPORTED   :   return "CUSPARSE_STATUS_MATRIX_TYPE_NOT_SUPPORTED"  ;
-        case    CUSPARSE_STATUS_ZERO_PIVOT                  :   return "CUSPARSE_STATUS_ZERO_PIVOT"                 ;
-        default                                             :   return "UNKNOWN";
-        }
+namespace cusparse
+{
+
+const char *errorString(cusparseStatus_t err)
+{
+    switch(err) {
+    case    CUSPARSE_STATUS_SUCCESS                     :   return "CUSPARSE_STATUS_SUCCESS"                    ;
+    case    CUSPARSE_STATUS_NOT_INITIALIZED             :   return "CUSPARSE_STATUS_NOT_INITIALIZED"            ;
+    case    CUSPARSE_STATUS_ALLOC_FAILED                :   return "CUSPARSE_STATUS_ALLOC_FAILED"               ;
+    case    CUSPARSE_STATUS_INVALID_VALUE               :   return "CUSPARSE_STATUS_INVALID_VALUE"              ;
+    case    CUSPARSE_STATUS_ARCH_MISMATCH               :   return "CUSPARSE_STATUS_ARCH_MISMATCH"              ;
+    case    CUSPARSE_STATUS_MAPPING_ERROR               :   return "CUSPARSE_STATUS_MAPPING_ERROR"              ;
+    case    CUSPARSE_STATUS_EXECUTION_FAILED            :   return "CUSPARSE_STATUS_EXECUTION_FAILED"           ;
+    case    CUSPARSE_STATUS_INTERNAL_ERROR              :   return "CUSPARSE_STATUS_INTERNAL_ERROR"             ;
+    case    CUSPARSE_STATUS_MATRIX_TYPE_NOT_SUPPORTED   :   return "CUSPARSE_STATUS_MATRIX_TYPE_NOT_SUPPORTED"  ;
+    case    CUSPARSE_STATUS_ZERO_PIVOT                  :   return "CUSPARSE_STATUS_ZERO_PIVOT"                 ;
+    default                                             :   return "UNKNOWN";
     }
+}
 
 
-//RAII class around the cusparse Handle
-    class cusparseHandle
-    {
-        cusparseHandle_t handle;
-    public:
-
-        cusparseHandle()
-            : handle(0)
-        {
-            CUSPARSE_CHECK(cusparseCreate(&handle));
-            CUSPARSE_CHECK(cusparseSetStream(handle, cuda::getStream(cuda::getActiveDeviceId())));
-        }
-
-        ~cusparseHandle()
-        {
-            cusparseDestroy(handle);
-        }
-
-        cusparseHandle_t get() const
-        {
-            return handle;
-        }
-    };
-
-    cusparseHandle_t getHandle()
-    {
-        using boost::scoped_ptr;
-        static scoped_ptr<cusparseHandle> handle[cuda::DeviceManager::MAX_DEVICES];
-
-        int id = cuda::getActiveDeviceId();
+cusparseHandle::cusparseHandle()
+    : handle(0)
+{
+    CUSPARSE_CHECK(cusparseCreate(&handle));
+    CUSPARSE_CHECK(cusparseSetStream(handle, cuda::getStream(cuda::getActiveDeviceId())));
+}
 
-        if(!handle[id]) {
-            handle[id].reset(new cusparseHandle());
-        }
+cusparseHandle::~cusparseHandle()
+{
+    cusparseDestroy(handle);
+}
 
-        return handle[id]->get();
-    }
+cusparseHandle_t cusparseHandle::get() const
+{
+    return handle;
+}
 
 }
diff --git a/src/backend/cuda/cusparseManager.hpp b/src/backend/cuda/cusparseManager.hpp
index 23fcbf12c9..b5b3625f18 100644
--- a/src/backend/cuda/cusparseManager.hpp
+++ b/src/backend/cuda/cusparseManager.hpp
@@ -13,10 +13,35 @@
 #include <defines.hpp>
 #include <cusparse_v2.h>
 
-namespace cusparse {
+namespace cuda
+{
+
+class DeviceManager;
+
+}
+
+namespace cusparse
+{
+
+const char * errorString(cusparseStatus_t err);
+
+//RAII class around the cusparse Handle
+class cusparseHandle
+{
+    friend class cuda::DeviceManager;
+
+    public:
+    ~cusparseHandle();
+    cusparseHandle_t get() const;
+
+    private:
+    cusparseHandle();
+    cusparseHandle(cusparseHandle const&);
+    void operator=(cusparseHandle const&);
+
+    cusparseHandle_t handle;
+};
 
-    const char * errorString(cusparseStatus_t err);
-    cusparseHandle_t getHandle();
 }
 
 
diff --git a/src/backend/cuda/platform.cpp b/src/backend/cuda/platform.cpp
index 4e6d581e12..58def6f604 100644
--- a/src/backend/cuda/platform.cpp
+++ b/src/backend/cuda/platform.cpp
@@ -429,6 +429,18 @@ cusolverDnHandle_t getcusolverDnHandle()
     return instance.cusolverHandles[id]->get();
 }
 
+cusparseHandle_t getcusparseHandle()
+{
+    DeviceManager& instance = DeviceManager::getInstance();
+
+    int id = cuda::getActiveDeviceId();
+
+    if (!(instance.cusparseHandles[id]))
+        instance.resetcusparseHandle(id);
+
+    return instance.cusparseHandles[id]->get();
+}
+
 DeviceManager::DeviceManager()
     : cuDevices(0), activeDev(0), nDevices(0), gfxManager(new InteropManager())
 {
@@ -561,6 +573,11 @@ void DeviceManager::resetcusolverHandle(int device)
     cusolverHandles[device].reset(new cusolver::cusolverDnHandle());
 }
 
+void DeviceManager::resetcusparseHandle(int device)
+{
+    cusparseHandles[device].reset(new cusparse::cusparseHandle());
+}
+
 void sync(int device)
 {
     int currDevice = getActiveDeviceId();
diff --git a/src/backend/cuda/platform.hpp b/src/backend/cuda/platform.hpp
index e87a843f0b..5c861e7744 100644
--- a/src/backend/cuda/platform.hpp
+++ b/src/backend/cuda/platform.hpp
@@ -22,6 +22,7 @@
 #include <cufftManager.hpp>
 #include <cublasManager.hpp>
 #include <cusolverDnManager.hpp>
+#include <cusparseManager.hpp>
 
 namespace cuda
 {
@@ -83,6 +84,8 @@ cufft::cuFFTPlanner& getcufftPlanManager();
 cublasHandle_t getcublasHandle();
 
 cusolverDnHandle_t getcusolverDnHandle();
+
+cusparseHandle_t getcusparseHandle();
 //
 ///////////////////////// END Sub-Managers /////////////////////
 
@@ -103,6 +106,8 @@ class DeviceManager
 
         friend cusolverDnHandle_t getcusolverDnHandle();
 
+        friend cusparseHandle_t getcusparseHandle();
+
         friend std::string getDeviceInfo(int device);
 
         friend std::string getPlatformInfo();
@@ -150,6 +155,8 @@ class DeviceManager
 
         void resetcusolverHandle(int device);
 
+        void resetcusparseHandle(int device);
+
         int activeDev;
         int nDevices;
         cudaStream_t streams[MAX_DEVICES];
@@ -161,6 +168,8 @@ class DeviceManager
         boost::scoped_ptr<cublas::cublasHandle> cublasHandles[MAX_DEVICES];
 
         boost::scoped_ptr<cusolver::cusolverDnHandle> cusolverHandles[MAX_DEVICES];
+
+        boost::scoped_ptr<cusparse::cusparseHandle> cusparseHandles[MAX_DEVICES];
 };
 
 }
diff --git a/src/backend/cuda/sparse.cu b/src/backend/cuda/sparse.cu
index 1d13f94ed0..6a2402e33e 100644
--- a/src/backend/cuda/sparse.cu
+++ b/src/backend/cuda/sparse.cu
@@ -8,7 +8,6 @@
  ********************************************************/
 
 #include <sparse.hpp>
-#include <cusparseManager.hpp>
 #include <kernel/sparse.hpp>
 
 #include <stdexcept>
@@ -29,7 +28,6 @@
 namespace cuda
 {
 
-using cusparse::getHandle;
 using namespace common;
 using namespace std;
 
@@ -265,7 +263,7 @@ SparseArray<T> sparseConvertDenseToStorage(const Array<T> &in)
 
     int nNZ = -1;
     CUSPARSE_CHECK(nnz_func<T>()(
-                        getHandle(),
+                        getcusparseHandle(),
                         dir,
                         M, N,
                         descr,
@@ -286,7 +284,7 @@ SparseArray<T> sparseConvertDenseToStorage(const Array<T> &in)
 
     if(stype == AF_STORAGE_CSR)
         CUSPARSE_CHECK(dense2csr_func<T>()(
-                        getHandle(),
+                        getcusparseHandle(),
                         M, N,
                         descr,
                         in.get(), in.strides()[1],
@@ -294,7 +292,7 @@ SparseArray<T> sparseConvertDenseToStorage(const Array<T> &in)
                         values.get(), rowIdx.get(), colIdx.get()));
     else
         CUSPARSE_CHECK(dense2csc_func<T>()(
-                        getHandle(),
+                        getcusparseHandle(),
                         M, N,
                         descr,
                         in.get(), in.strides()[1],
@@ -340,7 +338,7 @@ Array<T> sparseConvertStorageToDense(const SparseArray<T> &in)
 
     if(stype == AF_STORAGE_CSR)
         CUSPARSE_CHECK(csr2dense_func<T>()(
-                        getHandle(),
+                        getcusparseHandle(),
                         M, N,
                         descr,
                         in.getValues().get(),
@@ -349,7 +347,7 @@ Array<T> sparseConvertStorageToDense(const SparseArray<T> &in)
                         dense.get(), d_strides1));
     else
         CUSPARSE_CHECK(csc2dense_func<T>()(
-                        getHandle(),
+                        getcusparseHandle(),
                         M, N,
                         descr,
                         in.getValues().get(),
@@ -383,7 +381,7 @@ SparseArray<T> sparseConvertStorageToStorage(const SparseArray<T> &in)
 
         // cusparse function to expand compressed row into coordinate
         CUSPARSE_CHECK(cusparseXcsr2coo(
-                        getHandle(),
+                        getcusparseHandle(),
                         in.getRowIdx().get(),
                         nNZ, in.dims()[0],
                         converted.getRowIdx().get(),
@@ -392,23 +390,23 @@ SparseArray<T> sparseConvertStorageToStorage(const SparseArray<T> &in)
         // Call sort
         size_t pBufferSizeInBytes = 0;
         CUSPARSE_CHECK(cusparseXcoosort_bufferSizeExt(
-                        getHandle(),
+                        getcusparseHandle(),
                         in.dims()[0], in.dims()[1], nNZ,
                         converted.getRowIdx().get(), converted.getColIdx().get(),
                         &pBufferSizeInBytes));
         shared_ptr<char> pBuffer(memAlloc<char>(pBufferSizeInBytes), memFree<char>);
 
         shared_ptr<int> P(memAlloc<int>(nNZ), memFree<int>);
-        CUSPARSE_CHECK(cusparseCreateIdentityPermutation(getHandle(), nNZ, P.get()));
+        CUSPARSE_CHECK(cusparseCreateIdentityPermutation(getcusparseHandle(), nNZ, P.get()));
 
         CUSPARSE_CHECK(cusparseXcoosortByColumn(
-                       getHandle(),
+                       getcusparseHandle(),
                        in.dims()[0], in.dims()[1], nNZ,
                        converted.getRowIdx().get(), converted.getColIdx().get(),
                        P.get(), (void*)pBuffer.get()));
 
         CUSPARSE_CHECK(gthr_func<T>()(
-                       getHandle(), nNZ,
+                       getcusparseHandle(), nNZ,
                        in.getValues().get(),
                        converted.getValues().get(),
                        P.get(), CUSPARSE_INDEX_BASE_ZERO));
@@ -427,23 +425,23 @@ SparseArray<T> sparseConvertStorageToStorage(const SparseArray<T> &in)
         {
             size_t pBufferSizeInBytes = 0;
             CUSPARSE_CHECK(cusparseXcoosort_bufferSizeExt(
-                            getHandle(),
+                            getcusparseHandle(),
                             cooT.dims()[0], cooT.dims()[1], nNZ,
                             cooT.getRowIdx().get(), cooT.getColIdx().get(),
                             &pBufferSizeInBytes));
             shared_ptr<char> pBuffer(memAlloc<char>(pBufferSizeInBytes), memFree<char>);
 
             shared_ptr<int> P(memAlloc<int>(nNZ), memFree<int>);
-            CUSPARSE_CHECK(cusparseCreateIdentityPermutation(getHandle(), nNZ, P.get()));
+            CUSPARSE_CHECK(cusparseCreateIdentityPermutation(getcusparseHandle(), nNZ, P.get()));
 
             CUSPARSE_CHECK(cusparseXcoosortByRow(
-                           getHandle(),
+                           getcusparseHandle(),
                            cooT.dims()[0], cooT.dims()[1], nNZ,
                            cooT.getRowIdx().get(), cooT.getColIdx().get(),
                            P.get(), (void*)pBuffer.get()));
 
             CUSPARSE_CHECK(gthr_func<T>()(
-                           getHandle(), nNZ,
+                           getcusparseHandle(), nNZ,
                            in.getValues().get(),
                            cooT.getValues().get(),
                            P.get(), CUSPARSE_INDEX_BASE_ZERO));
@@ -462,7 +460,7 @@ SparseArray<T> sparseConvertStorageToStorage(const SparseArray<T> &in)
 
         // cusparse function to compress row from coordinate
         CUSPARSE_CHECK(cusparseXcoo2csr(
-                        getHandle(),
+                        getcusparseHandle(),
                         cooT.getRowIdx().get(),
                         nNZ, cooT.dims()[0],
                         converted.getRowIdx().get(),
diff --git a/src/backend/cuda/sparse_blas.cpp b/src/backend/cuda/sparse_blas.cpp
index a8394abdcb..1d6dc27775 100644
--- a/src/backend/cuda/sparse_blas.cpp
+++ b/src/backend/cuda/sparse_blas.cpp
@@ -8,7 +8,6 @@
  ********************************************************/
 
 #include <sparse_blas.hpp>
-#include <cusparseManager.hpp>
 #include <cuda_runtime.h>
 #include <platform.hpp>
 
@@ -21,7 +20,6 @@
 namespace cuda
 {
 
-using cusparse::getHandle;
 using namespace std;
 
 cusparseOperation_t
@@ -157,7 +155,7 @@ Array<T> matmul(const common::SparseArray<T> lhs, const Array<T> rhs,
     // and not OP(A) (gemm wants row/col of OP(A)).
     if(rDims[rColDim] == 1) {
         CUSPARSE_CHECK(csrmv_func<T>()(
-                       getHandle(),
+                       getcusparseHandle(),
                        lOpts,
                        lDims[0], lDims[1], lhs.getNNZ(),
                        &alpha,
@@ -168,7 +166,7 @@ Array<T> matmul(const common::SparseArray<T> lhs, const Array<T> rhs,
                        out.get()));
     } else {
         CUSPARSE_CHECK(csrmm_func<T>()(
-                       getHandle(),
+                       getcusparseHandle(),
                        lOpts,
                        lDims[0], rDims[rColDim], lDims[1], lhs.getNNZ(),
                        &alpha,

From 8de2a5911bc3d4d2d03a1fd6644718d82075d6e3 Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Fri, 23 Dec 2016 07:15:16 +0530
Subject: [PATCH 09/82] Merge cuda memory manager into cuda::DeviceManager

---
 src/backend/cuda/memory.cpp        | 135 +----------------------------
 src/backend/cuda/memoryManager.cpp |  86 ++++++++++++++++++
 src/backend/cuda/memoryManager.hpp |  63 ++++++++++++++
 src/backend/cuda/platform.cpp      |  11 +++
 src/backend/cuda/platform.hpp      |  14 +++
 5 files changed, 175 insertions(+), 134 deletions(-)
 create mode 100644 src/backend/cuda/memoryManager.cpp
 create mode 100644 src/backend/cuda/memoryManager.hpp

diff --git a/src/backend/cuda/memory.cpp b/src/backend/cuda/memory.cpp
index 03ebc20d44..9b2eb07c44 100644
--- a/src/backend/cuda/memory.cpp
+++ b/src/backend/cuda/memory.cpp
@@ -8,147 +8,14 @@
  ********************************************************/
 
 #include <memory.hpp>
-#include <cuda.h>
-#include <cuda_runtime_api.h>
-#include <cuda_runtime.h>
-#include <err_cuda.hpp>
 #include <util.hpp>
 #include <types.hpp>
-#include <iostream>
-#include <iomanip>
-#include <string>
-#include <map>
 #include <dispatch.hpp>
-#include <platform.hpp>
-#include <MemoryManager.hpp>
-
-
-#ifndef AF_MEM_DEBUG
-#define AF_MEM_DEBUG 0
-#endif
-
-#ifndef AF_CUDA_MEM_DEBUG
-#define AF_CUDA_MEM_DEBUG 0
-#endif
+#include <memoryManager.hpp>
 
 namespace cuda
 {
 
-class MemoryManager  : public common::MemoryManager
-{
-    int getActiveDeviceId();
-    size_t getMaxMemorySize(int id);
-public:
-    MemoryManager();
-    void *nativeAlloc(const size_t bytes);
-    void nativeFree(void *ptr);
-    ~MemoryManager()
-    {
-        common::lock_guard_t lock(this->memory_mutex);
-        for (int n = 0; n < getDeviceCount(); n++) {
-            try {
-                cuda::setDevice(n);
-                this->garbageCollect();
-            } catch(AfError err) {
-                continue; // Do not throw any errors while shutting down
-            }
-        }
-    }
-};
-
-// CUDA Pinned Memory does not depend on device
-// So we pass 1 as numDevices to the constructor so that it creates 1 vector
-// of memory_info
-// When allocating and freeing, it doesn't really matter which device is active
-class MemoryManagerPinned  : public common::MemoryManager
-{
-    int getActiveDeviceId();
-    size_t getMaxMemorySize(int id);
-public:
-    MemoryManagerPinned();
-    void *nativeAlloc(const size_t bytes);
-    void nativeFree(void *ptr);
-    ~MemoryManagerPinned()
-    {
-        common::lock_guard_t lock(this->memory_mutex);
-        this->garbageCollect();
-    }
-};
-
-int MemoryManager::getActiveDeviceId()
-{
-    return cuda::getActiveDeviceId();
-}
-
-size_t MemoryManager::getMaxMemorySize(int id)
-{
-    return cuda::getDeviceMemorySize(id);
-}
-
-MemoryManager::MemoryManager() :
-    common::MemoryManager(getDeviceCount(), common::MAX_BUFFERS, AF_MEM_DEBUG || AF_CUDA_MEM_DEBUG)
-{
-    this->setMaxMemorySize();
-}
-
-void *MemoryManager::nativeAlloc(const size_t bytes)
-{
-    void *ptr = NULL;
-    CUDA_CHECK(cudaMalloc(&ptr, bytes));
-    return ptr;
-}
-
-void MemoryManager::nativeFree(void *ptr)
-{
-    cudaError_t err = cudaFree(ptr);
-    if (err != cudaErrorCudartUnloading) {
-        CUDA_CHECK(err);
-    }
-}
-
-static MemoryManager &getMemoryManager()
-{
-    static MemoryManager instance;
-    return instance;
-}
-
-int MemoryManagerPinned::getActiveDeviceId()
-{
-    return 0; // pinned uses a single vector
-}
-
-size_t MemoryManagerPinned::getMaxMemorySize(int id)
-{
-    return cuda::getHostMemorySize();
-}
-
-MemoryManagerPinned::MemoryManagerPinned() :
-    common::MemoryManager(1, common::MAX_BUFFERS, AF_MEM_DEBUG || AF_CUDA_MEM_DEBUG)
-{
-    this->setMaxMemorySize();
-}
-
-void *MemoryManagerPinned::nativeAlloc(const size_t bytes)
-{
-    void *ptr;
-    CUDA_CHECK(cudaMallocHost(&ptr, bytes));
-    return ptr;
-}
-
-void MemoryManagerPinned::nativeFree(void *ptr)
-{
-    cudaError_t err = cudaFreeHost(ptr);
-    if (err != cudaErrorCudartUnloading) {
-        CUDA_CHECK(err);
-    }
-}
-
-static MemoryManagerPinned &getMemoryManagerPinned()
-{
-    static MemoryManagerPinned instance;
-    return instance;
-}
-
 void setMemStepSize(size_t step_bytes)
 {
     getMemoryManager().setMemStepSize(step_bytes);
diff --git a/src/backend/cuda/memoryManager.cpp b/src/backend/cuda/memoryManager.cpp
new file mode 100644
index 0000000000..d160471814
--- /dev/null
+++ b/src/backend/cuda/memoryManager.cpp
@@ -0,0 +1,86 @@
+/*******************************************************
+ * Copyright (c) 2016, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <err_cuda.hpp>
+#include <memoryManager.hpp>
+
+#ifndef AF_MEM_DEBUG
+#define AF_MEM_DEBUG 0
+#endif
+
+#ifndef AF_CUDA_MEM_DEBUG
+#define AF_CUDA_MEM_DEBUG 0
+#endif
+
+namespace cuda
+{
+
+int MemoryManager::getActiveDeviceId()
+{
+    return cuda::getActiveDeviceId();
+}
+
+size_t MemoryManager::getMaxMemorySize(int id)
+{
+    return cuda::getDeviceMemorySize(id);
+}
+
+MemoryManager::MemoryManager() :
+    common::MemoryManager(getDeviceCount(), common::MAX_BUFFERS, AF_MEM_DEBUG || AF_CUDA_MEM_DEBUG)
+{
+    this->setMaxMemorySize();
+}
+
+void *MemoryManager::nativeAlloc(const size_t bytes)
+{
+    void *ptr = NULL;
+    CUDA_CHECK(cudaMalloc(&ptr, bytes));
+    return ptr;
+}
+
+void MemoryManager::nativeFree(void *ptr)
+{
+    cudaError_t err = cudaFree(ptr);
+    if (err != cudaErrorCudartUnloading) {
+        CUDA_CHECK(err);
+    }
+}
+
+int MemoryManagerPinned::getActiveDeviceId()
+{
+    return 0; // pinned uses a single vector
+}
+
+size_t MemoryManagerPinned::getMaxMemorySize(int id)
+{
+    return cuda::getHostMemorySize();
+}
+
+MemoryManagerPinned::MemoryManagerPinned() :
+    common::MemoryManager(1, common::MAX_BUFFERS, AF_MEM_DEBUG || AF_CUDA_MEM_DEBUG)
+{
+    this->setMaxMemorySize();
+}
+
+void *MemoryManagerPinned::nativeAlloc(const size_t bytes)
+{
+    void *ptr;
+    CUDA_CHECK(cudaMallocHost(&ptr, bytes));
+    return ptr;
+}
+
+void MemoryManagerPinned::nativeFree(void *ptr)
+{
+    cudaError_t err = cudaFreeHost(ptr);
+    if (err != cudaErrorCudartUnloading) {
+        CUDA_CHECK(err);
+    }
+}
+
+}
diff --git a/src/backend/cuda/memoryManager.hpp b/src/backend/cuda/memoryManager.hpp
new file mode 100644
index 0000000000..b22bd02c1f
--- /dev/null
+++ b/src/backend/cuda/memoryManager.hpp
@@ -0,0 +1,63 @@
+/*******************************************************
+ * Copyright (c) 2016, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <cuda.h>
+#include <cuda_runtime_api.h>
+#include <cuda_runtime.h>
+
+#include <platform.hpp>
+#include <MemoryManager.hpp>
+
+namespace cuda
+{
+
+class MemoryManager  : public common::MemoryManager
+{
+    int getActiveDeviceId();
+    size_t getMaxMemorySize(int id);
+public:
+    MemoryManager();
+    void *nativeAlloc(const size_t bytes);
+    void nativeFree(void *ptr);
+    ~MemoryManager()
+    {
+        common::lock_guard_t lock(this->memory_mutex);
+        for (int n = 0; n < getDeviceCount(); n++) {
+            try {
+                cuda::setDevice(n);
+                this->garbageCollect();
+            } catch(AfError err) {
+                continue; // Do not throw any errors while shutting down
+            }
+        }
+    }
+};
+
+// CUDA Pinned Memory does not depend on device
+// So we pass 1 as numDevices to the constructor so that it creates 1 vector
+// of memory_info
+// When allocating and freeing, it doesn't really matter which device is active
+class MemoryManagerPinned  : public common::MemoryManager
+{
+    int getActiveDeviceId();
+    size_t getMaxMemorySize(int id);
+public:
+    MemoryManagerPinned();
+    void *nativeAlloc(const size_t bytes);
+    void nativeFree(void *ptr);
+    ~MemoryManagerPinned()
+    {
+        common::lock_guard_t lock(this->memory_mutex);
+        this->garbageCollect();
+    }
+};
+
+}
diff --git a/src/backend/cuda/platform.cpp b/src/backend/cuda/platform.cpp
index 58def6f604..9e50865025 100644
--- a/src/backend/cuda/platform.cpp
+++ b/src/backend/cuda/platform.cpp
@@ -24,6 +24,7 @@
 #include <err_cuda.hpp>
 #include <util.hpp>
 #include <host_memory.hpp>
+#include <memoryManager.hpp>
 #include <interopManager.hpp>
 
 using namespace std;
@@ -380,6 +381,16 @@ DeviceManager& DeviceManager::getInstance()
     return my_instance;
 }
 
+MemoryManager &getMemoryManager()
+{
+    return *(DeviceManager::getInstance().memManager.get());
+}
+
+MemoryManagerPinned &getMemoryManagerPinned()
+{
+    return *(DeviceManager::getInstance().pinnedMemManager.get());
+}
+
 InteropManager& getGfxInteropManager()
 {
     return *(DeviceManager::getInstance().gfxManager.get());
diff --git a/src/backend/cuda/platform.hpp b/src/backend/cuda/platform.hpp
index 5c861e7744..0e82653991 100644
--- a/src/backend/cuda/platform.hpp
+++ b/src/backend/cuda/platform.hpp
@@ -73,10 +73,16 @@ struct cudaDevice_t {
 
 bool& evalFlag();
 
+class MemoryManager;
+class MemoryManagerPinned;
 class InteropManager;
 
 ///////////////////////// BEGIN Sub-Managers ///////////////////
 //
+MemoryManager& getMemoryManager();
+
+MemoryManagerPinned& getMemoryManagerPinned();
+
 InteropManager& getGfxInteropManager();
 
 cufft::cuFFTPlanner& getcufftPlanManager();
@@ -98,6 +104,10 @@ class DeviceManager
 
         static DeviceManager& getInstance();
 
+        friend MemoryManager& getMemoryManager();
+
+        friend MemoryManagerPinned& getMemoryManagerPinned();
+
         friend InteropManager& getGfxInteropManager();
 
         friend cufft::cuFFTPlanner& getcufftPlanManager();
@@ -161,6 +171,10 @@ class DeviceManager
         int nDevices;
         cudaStream_t streams[MAX_DEVICES];
 
+        boost::scoped_ptr<MemoryManager> memManager;
+
+        boost::scoped_ptr<MemoryManagerPinned> pinnedMemManager;
+
         boost::scoped_ptr<InteropManager> gfxManager;
 
         cufft::cuFFTPlanner cufftManagers[MAX_DEVICES];

From b204faf04688203088367e4fc2117eadd015d111 Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Fri, 23 Dec 2016 15:06:25 +0530
Subject: [PATCH 10/82] fix cuda backend sub-managers initilization order

---
 src/backend/cuda/platform.cpp | 23 +++++++++++++++++++----
 1 file changed, 19 insertions(+), 4 deletions(-)

diff --git a/src/backend/cuda/platform.cpp b/src/backend/cuda/platform.cpp
index 9e50865025..bce94b4945 100644
--- a/src/backend/cuda/platform.cpp
+++ b/src/backend/cuda/platform.cpp
@@ -383,17 +383,32 @@ DeviceManager& DeviceManager::getInstance()
 
 MemoryManager &getMemoryManager()
 {
-    return *(DeviceManager::getInstance().memManager.get());
+    DeviceManager& inst = DeviceManager::getInstance();
+
+    if (!inst.memManager.get())
+        inst.memManager.reset(new cuda::MemoryManager());
+
+    return *(inst.memManager.get());
 }
 
 MemoryManagerPinned &getMemoryManagerPinned()
 {
-    return *(DeviceManager::getInstance().pinnedMemManager.get());
+    DeviceManager& inst = DeviceManager::getInstance();
+
+    if (!inst.pinnedMemManager.get())
+        inst.pinnedMemManager.reset(new cuda::MemoryManagerPinned());
+
+    return *(inst.pinnedMemManager.get());
 }
 
 InteropManager& getGfxInteropManager()
 {
-    return *(DeviceManager::getInstance().gfxManager.get());
+    DeviceManager& inst = DeviceManager::getInstance();
+
+    if (!inst.gfxManager.get())
+        inst.gfxManager.reset(new cuda::InteropManager());
+
+    return *(inst.gfxManager.get());
 }
 
 cufft::cuFFTPlanner& getcufftPlanManager()
@@ -453,7 +468,7 @@ cusparseHandle_t getcusparseHandle()
 }
 
 DeviceManager::DeviceManager()
-    : cuDevices(0), activeDev(0), nDevices(0), gfxManager(new InteropManager())
+    : cuDevices(0), activeDev(0), nDevices(0)
 {
     CUDA_CHECK(cudaGetDeviceCount(&nDevices));
     if (nDevices == 0)

From 211757166a97d6c6847c286ee17686c0ec5bdbab Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Fri, 23 Dec 2016 17:53:42 +0530
Subject: [PATCH 11/82] Added locks for memory manager initialization

---
 src/backend/cuda/CMakeLists.txt |  5 +++--
 src/backend/cuda/platform.cpp   | 34 +++++++++++++++++++++++++++++----
 src/backend/cuda/platform.hpp   |  3 +++
 3 files changed, 36 insertions(+), 6 deletions(-)

diff --git a/src/backend/cuda/CMakeLists.txt b/src/backend/cuda/CMakeLists.txt
index 9cfa8a5929..424bda9a0e 100644
--- a/src/backend/cuda/CMakeLists.txt
+++ b/src/backend/cuda/CMakeLists.txt
@@ -1,7 +1,7 @@
 CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 
 FIND_PACKAGE(CUDA REQUIRED)
-FIND_PACKAGE(Boost REQUIRED)
+FIND_PACKAGE(Boost REQUIRED COMPONENTS "system" "thread")
 
 INCLUDE(CLKernelToH)
 INCLUDE(FindNVVM)
@@ -498,7 +498,8 @@ TARGET_LINK_LIBRARIES(afcuda    PRIVATE ${CUDA_CUBLAS_LIBRARIES}
                                 PRIVATE ${CUDA_CUFFT_LIBRARIES}
                                 PRIVATE ${CUDA_cusparse_LIBRARY}
                                 PRIVATE ${CUDA_nvvm_LIBRARY}
-                                PRIVATE ${CUDA_CUDA_LIBRARY})
+                                PRIVATE ${CUDA_CUDA_LIBRARY}
+                                PRIVATE ${Boost_LIBRARIES})
 
 LIST(LENGTH GRAPHICS_DEPENDENCIES GRAPHICS_DEPENDENCIES_LEN)
 IF(${GRAPHICS_DEPENDENCIES_LEN} GREATER 0)
diff --git a/src/backend/cuda/platform.cpp b/src/backend/cuda/platform.cpp
index bce94b4945..afdbe3eaea 100644
--- a/src/backend/cuda/platform.cpp
+++ b/src/backend/cuda/platform.cpp
@@ -385,8 +385,21 @@ MemoryManager &getMemoryManager()
 {
     DeviceManager& inst = DeviceManager::getInstance();
 
-    if (!inst.memManager.get())
-        inst.memManager.reset(new cuda::MemoryManager());
+    boost::upgrade_lock<boost::shared_mutex> lock(inst.memManagerMutex);
+
+    // multiple threads should be able to reach the
+    // conditional statement without contention
+    if (!inst.memManager.get()) {
+        // upgrade the shared ownership we acquired earlier to
+        // exclusive owner ship to initialize the memory manager
+        boost::upgrade_to_unique_lock<boost::shared_mutex> unqLock(lock);
+
+        // if multiple threads above statment, allow the
+        // initialization of memory manager to happen once
+        // by checking if pinned memory manager pointer is already set
+        if (!inst.memManager.get())
+            inst.memManager.reset(new cuda::MemoryManager());
+    }
 
     return *(inst.memManager.get());
 }
@@ -395,8 +408,21 @@ MemoryManagerPinned &getMemoryManagerPinned()
 {
     DeviceManager& inst = DeviceManager::getInstance();
 
-    if (!inst.pinnedMemManager.get())
-        inst.pinnedMemManager.reset(new cuda::MemoryManagerPinned());
+    boost::upgrade_lock<boost::shared_mutex> lock(inst.memManagerMutex);
+
+    // multiple threads should be able to reach the
+    // conditional statement without contention
+    if (!inst.pinnedMemManager.get()) {
+        // upgrade the shared ownership we acquired earlier to
+        // exclusive owner ship to initialize the memory manager
+        boost::upgrade_to_unique_lock<boost::shared_mutex> unqLock(lock);
+
+        // if multiple threads above statment, allow the
+        // initialization of memory manager to happen once
+        // by checking if pinned memory manager pointer is already set
+        if (!inst.pinnedMemManager.get())
+            inst.pinnedMemManager.reset(new cuda::MemoryManagerPinned());
+    }
 
     return *(inst.pinnedMemManager.get());
 }
diff --git a/src/backend/cuda/platform.hpp b/src/backend/cuda/platform.hpp
index 0e82653991..7264c5aa16 100644
--- a/src/backend/cuda/platform.hpp
+++ b/src/backend/cuda/platform.hpp
@@ -18,6 +18,7 @@
 #endif
 
 #include <boost/scoped_ptr.hpp>
+#include <boost/thread/shared_mutex.hpp>
 
 #include <cufftManager.hpp>
 #include <cublasManager.hpp>
@@ -171,8 +172,10 @@ class DeviceManager
         int nDevices;
         cudaStream_t streams[MAX_DEVICES];
 
+        boost::shared_mutex memManagerMutex;
         boost::scoped_ptr<MemoryManager> memManager;
 
+        boost::shared_mutex pinnedMemManagerMutex;
         boost::scoped_ptr<MemoryManagerPinned> pinnedMemManager;
 
         boost::scoped_ptr<InteropManager> gfxManager;

From 15cc0183396dbaf0a7a284ce3747eddf37533e33 Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Mon, 9 Jan 2017 12:58:52 +0530
Subject: [PATCH 12/82] Fix for CUDA usage with gcc > 5.x

---
 src/backend/cuda/CMakeLists.txt | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/backend/cuda/CMakeLists.txt b/src/backend/cuda/CMakeLists.txt
index 9e4b5090b9..4da7cec1be 100644
--- a/src/backend/cuda/CMakeLists.txt
+++ b/src/backend/cuda/CMakeLists.txt
@@ -338,6 +338,10 @@ ELSE()
   ENDIF()
 ENDIF()
 
+IF("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
+    SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_MWAITXINTRIN_H_INCLUDED -D_FORCE_INLINES")
+ENDIF()
+
 ## Copied from FindCUDA.cmake
 ## The target_link_library needs to link with the cuda libraries using
 ## PRIVATE

From aaca5f9ff5eb1030c15cf5d931fe79f438cf147d Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Mon, 9 Jan 2017 18:02:59 +0530
Subject: [PATCH 13/82] Move clfft intialization into opencl::DeviceManager

---
 .../{err_clfft.hpp => clfftManager.cpp}       | 154 +++++++++++---
 src/backend/opencl/clfftManager.hpp           | 127 ++++++++++++
 src/backend/opencl/fft.cpp                    | 193 +-----------------
 src/backend/opencl/memory.hpp                 |   7 +-
 src/backend/opencl/platform.cpp               |   5 +
 src/backend/opencl/platform.hpp               |  13 ++
 6 files changed, 285 insertions(+), 214 deletions(-)
 rename src/backend/opencl/{err_clfft.hpp => clfftManager.cpp} (52%)
 create mode 100644 src/backend/opencl/clfftManager.hpp

diff --git a/src/backend/opencl/err_clfft.hpp b/src/backend/opencl/clfftManager.cpp
similarity index 52%
rename from src/backend/opencl/err_clfft.hpp
rename to src/backend/opencl/clfftManager.cpp
index 94ca210c46..ef08442474 100644
--- a/src/backend/opencl/err_clfft.hpp
+++ b/src/backend/opencl/clfftManager.cpp
@@ -1,5 +1,5 @@
 /*******************************************************
- * Copyright (c) 2014, ArrayFire
+ * Copyright (c) 2016, ArrayFire
  * All rights reserved.
  *
  * This file is distributed under 3-clause BSD license.
@@ -7,12 +7,20 @@
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
 
-#pragma once
-#include <stdio.h>
-#include <err_opencl.hpp>
-#include <clFFT.h>
+#include <af/defines.h>
 
-static const char * _clfftGetResultString(clfftStatus st)
+#include <err_common.hpp>
+#include <clfftManager.hpp>
+#include <platform.hpp>
+
+#include <string>
+
+using std::string;
+
+namespace clfft
+{
+
+const char * _clfftGetResultString(clfftStatus st)
 {
     switch (st)
     {
@@ -78,23 +86,119 @@ static const char * _clfftGetResultString(clfftStatus st)
     return "Unknown error";
 }
 
-#define CLFFT_CHECK(fn) do {                    \
-        clfftStatus _clfft_st = fn;             \
-        if (_clfft_st != CLFFT_SUCCESS) {       \
-            garbageCollect();                   \
-            _clfft_st = (fn);                   \
-        }                                       \
-        if (_clfft_st != CLFFT_SUCCESS) {       \
-            char clfft_st_msg[1024];            \
-            snprintf(clfft_st_msg,              \
-                     sizeof(clfft_st_msg),      \
-                     "clFFT Error (%d): %s\n",  \
-                     (int)(_clfft_st),          \
-                     _clfftGetResultString(     \
-                         _clfft_st));           \
-                                                \
-            AF_ERROR(clfft_st_msg,              \
-                     AF_ERR_INTERNAL);          \
-        }                                       \
-    } while(0)
+void findPlan(clfftPlanHandle &plan,
+              clfftLayout iLayout, clfftLayout oLayout,
+              clfftDim rank, size_t *clLengths,
+              size_t *istrides, size_t idist,
+              size_t *ostrides, size_t odist,
+              clfftPrecision precision, size_t batch)
+{
+    // create the key string
+    char key_str_temp[64];
+    sprintf(key_str_temp, "%d:%d:%d:", iLayout, oLayout, rank);
+
+    string key_string(key_str_temp);
+
+    /* WARNING: DO NOT CHANGE sprintf format specifier */
+    for(int r=0; r<rank; ++r) {
+        sprintf(key_str_temp, SIZE_T_FRMT_SPECIFIER ":", clLengths[r]);
+        key_string.append(std::string(key_str_temp));
+    }
+
+    if(istrides!=NULL) {
+        for(int r=0; r<rank; ++r) {
+            sprintf(key_str_temp, SIZE_T_FRMT_SPECIFIER ":", istrides[r]);
+            key_string.append(std::string(key_str_temp));
+        }
+        sprintf(key_str_temp, SIZE_T_FRMT_SPECIFIER ":", idist);
+        key_string.append(std::string(key_str_temp));
+    }
+
+    if (ostrides!=NULL) {
+        for(int r=0; r<rank; ++r) {
+            sprintf(key_str_temp, SIZE_T_FRMT_SPECIFIER ":", ostrides[r]);
+            key_string.append(std::string(key_str_temp));
+        }
+        sprintf(key_str_temp, SIZE_T_FRMT_SPECIFIER ":", odist);
+        key_string.append(std::string(key_str_temp));
+    }
+
+    sprintf(key_str_temp, "%d:" SIZE_T_FRMT_SPECIFIER, (int)precision, batch);
+    key_string.append(std::string(key_str_temp));
+
+    // find the matching plan_index in the array clFFTPlanner::mKeys
+    clFFTPlanner &planner = opencl::getclfftPlanManager();
+
+    int planIndex = planner.findIfPlanExists(key_string);
+
+    // if found a valid plan, return it
+    if (planIndex!=-1) {
+        plan = planner.getPlan(planIndex);
+        return;
+    }
+
+    clfftPlanHandle temp;
+
+    // getContext() returns object of type Context
+    // Context() returns the actual cl_context handle
+    CLFFT_CHECK(clfftCreateDefaultPlan(&temp, opencl::getContext()(), rank, clLengths));
 
+    // complex to complex
+    if (iLayout == oLayout) {
+        CLFFT_CHECK(clfftSetResultLocation(temp, CLFFT_INPLACE));
+    } else {
+        CLFFT_CHECK(clfftSetResultLocation(temp, CLFFT_OUTOFPLACE));
+    }
+
+    CLFFT_CHECK(clfftSetLayout(temp, iLayout, oLayout));
+    CLFFT_CHECK(clfftSetPlanBatchSize(temp, batch));
+    CLFFT_CHECK(clfftSetPlanDistance(temp, idist, odist));
+    CLFFT_CHECK(clfftSetPlanInStride(temp, rank, istrides));
+    CLFFT_CHECK(clfftSetPlanOutStride(temp, rank, ostrides));
+    CLFFT_CHECK(clfftSetPlanPrecision(temp, precision));
+    CLFFT_CHECK(clfftSetPlanScale(temp, CLFFT_BACKWARD, 1.0));
+
+    // getQueue() returns object of type CommandQueue
+    // CommandQueue() returns the actual cl_command_queue handle
+    CLFFT_CHECK(clfftBakePlan(temp, 1, &(opencl::getQueue()()), NULL, NULL));
+
+    plan = temp;
+
+    // push the plan into plan cache
+    planner.pushPlan(key_string, plan);
+}
+
+clFFTPlanner::clFFTPlanner()
+    : mMaxCacheSize(5)
+{
+    CLFFT_CHECK(clfftInitSetupData(&mFFTSetup));
+    CLFFT_CHECK(clfftSetup(&mFFTSetup));
+}
+
+clFFTPlanner::~clFFTPlanner()
+{
+    //TODO: FIXME:
+    // clfftTeardown() causes a "Pure Virtual Function Called" crash on
+    // Windows for Intel devices. This causes tests to fail.
+    #ifndef OS_WIN
+    static bool flag = true;
+    if(flag) {
+        // THOU SHALL NOT THROW IN DESTRUCTORS
+        clfftTeardown();
+        flag = false;
+    }
+    #endif
+}
+
+void clFFTPlanner::popPlan()
+{
+    if (!mCache.empty()) {
+        // destroy the clfft plan associated with the
+        // least recently used plan
+        CLFFT_CHECK(clfftDestroyPlan(&mCache.back().second));
+        // now pop the entry from cache
+        mCache.pop_back();
+    }
+}
+
+}
diff --git a/src/backend/opencl/clfftManager.hpp b/src/backend/opencl/clfftManager.hpp
new file mode 100644
index 0000000000..37ffd0fda0
--- /dev/null
+++ b/src/backend/opencl/clfftManager.hpp
@@ -0,0 +1,127 @@
+/*******************************************************
+ * Copyright (c) 2016, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <clFFT.h>
+
+#include <memory.hpp>
+
+#include <cstdio>
+#include <deque>
+#include <string>
+#include <utility>
+
+namespace opencl
+{
+class DeviceManager;
+}
+
+namespace clfft
+{
+
+typedef std::pair<std::string, clfftPlanHandle> FFTPlanPair;
+typedef std::deque<FFTPlanPair> FFTPlanCache;
+
+const char * _clfftGetResultString(clfftStatus st);
+
+void findPlan(clfftPlanHandle &plan,
+              clfftLayout iLayout, clfftLayout oLayout,
+              clfftDim rank, size_t *clLengths,
+              size_t *istrides, size_t idist,
+              size_t *ostrides, size_t odist,
+              clfftPrecision precision, size_t batch);
+
+// clFFTPlanner caches fft plans
+//
+// new plan |--> IF number of plans cached is at limit, pop the least used entry and push new plan.
+//          |
+//          |--> ELSE just push the plan
+// existing plan -> reuse a plan
+class clFFTPlanner
+{
+    friend class opencl::DeviceManager;
+
+    friend void findPlan(clfftPlanHandle &plan,
+                         clfftLayout iLayout, clfftLayout oLayout,
+                         clfftDim rank, size_t *clLengths,
+                         size_t *istrides, size_t idist,
+                         size_t *ostrides, size_t odist,
+                         clfftPrecision precision, size_t batch);
+
+    public:
+        clFFTPlanner();
+        ~clFFTPlanner();
+
+        inline void setMaxCacheSize(size_t size) {
+            mCache.resize(size, FFTPlanPair(std::string(""), 0));
+        }
+
+        inline size_t getMaxCacheSize() const {
+            return mMaxCacheSize;
+        }
+
+        inline clfftPlanHandle getPlan(int index) const {
+            return mCache[index].second;
+        }
+
+        // iterates through plan cache from front to back
+        // of the cache(queue)
+        int findIfPlanExists(std::string keyString) const {
+            int retVal = -1;
+            for(uint i=0; i<mCache.size(); ++i) {
+                if (keyString == mCache[i].first) {
+                    retVal = i;
+                }
+            }
+            return retVal;
+        }
+
+        // pops plan from the back of cache(queue)
+        void popPlan();
+
+        // pushes plan to the front of cache(queue)
+        void pushPlan(std::string keyString, clfftPlanHandle plan) {
+            if (mCache.size()>mMaxCacheSize) {
+                popPlan();
+            }
+            mCache.push_front(FFTPlanPair(keyString, plan));
+        }
+
+    private:
+        clFFTPlanner(clFFTPlanner const&);
+        void operator=(clFFTPlanner const&);
+
+        clfftSetupData  mFFTSetup;
+
+        size_t       mMaxCacheSize;
+        FFTPlanCache mCache;
+};
+
+}
+
+#define CLFFT_CHECK(fn) do {                        \
+        clfftStatus _clfft_st = fn;                 \
+        if (_clfft_st != CLFFT_SUCCESS) {           \
+            opencl::garbageCollect();               \
+            _clfft_st = (fn);                       \
+        }                                           \
+        if (_clfft_st != CLFFT_SUCCESS) {           \
+            char clfft_st_msg[1024];                \
+            snprintf(clfft_st_msg,                  \
+                     sizeof(clfft_st_msg),          \
+                     "clFFT Error (%d): %s\n",      \
+                     (int)(_clfft_st),              \
+                     clfft::_clfftGetResultString(  \
+                         _clfft_st));               \
+                                                    \
+            AF_ERROR(clfft_st_msg,                  \
+                     AF_ERR_INTERNAL);              \
+        }                                           \
+    } while(0)
diff --git a/src/backend/opencl/fft.cpp b/src/backend/opencl/fft.cpp
index 7cd39f46af..c5a550304d 100644
--- a/src/backend/opencl/fft.cpp
+++ b/src/backend/opencl/fft.cpp
@@ -12,14 +12,9 @@
 #include <copy.hpp>
 #include <fft.hpp>
 #include <err_opencl.hpp>
-#include <err_clfft.hpp>
-#include <clFFT.h>
 #include <math.hpp>
-#include <string>
-#include <deque>
-#include <utility>
-#include <cstdio>
 #include <memory.hpp>
+#include <clfftManager.hpp>
 
 using af::dim4;
 using std::string;
@@ -27,187 +22,9 @@ using std::string;
 namespace opencl
 {
 
-typedef std::pair<string, clfftPlanHandle> FFTPlanPair;
-typedef std::deque<FFTPlanPair> FFTPlanCache;
-
-// clFFTPlanner caches fft plans
-//
-// new plan |--> IF number of plans cached is at limit, pop the least used entry and push new plan.
-//          |
-//          |--> ELSE just push the plan
-// existing plan -> reuse a plan
-class clFFTPlanner
-{
-    friend void find_clfft_plan(clfftPlanHandle &plan,
-                                clfftLayout iLayout, clfftLayout oLayout,
-                                clfftDim rank, size_t *clLengths,
-                                size_t *istrides, size_t idist,
-                                size_t *ostrides, size_t odist,
-                                clfftPrecision precision, size_t batch);
-
-    public:
-        static clFFTPlanner& getInstance() {
-            static clFFTPlanner instances[opencl::DeviceManager::MAX_DEVICES];
-            return instances[opencl::getActiveDeviceId()];
-        }
-
-        ~clFFTPlanner() {
-            //TODO: FIXME:
-            // clfftTeardown() cause a "Pure Virtual Function Called" crash on
-            // Window only when Intel devices are called. This causes tests to
-            // fail.
-            #ifndef OS_WIN
-            static bool flag = true;
-            if(flag) {
-                // THOU SHALL NOT THROW IN DESTRUCTORS
-                clfftTeardown();
-                flag = false;
-            }
-            #endif
-        }
-
-        inline void setMaxCacheSize(size_t size) {
-            mCache.resize(size, FFTPlanPair(std::string(""), 0));
-        }
-
-        inline size_t getMaxCacheSize() const {
-            return mMaxCacheSize;
-        }
-
-        inline clfftPlanHandle getPlan(int index) const {
-            return mCache[index].second;
-        }
-
-        // iterates through plan cache from front to back
-        // of the cache(queue)
-        int findIfPlanExists(std::string keyString) const {
-            int retVal = -1;
-            for(uint i=0; i<mCache.size(); ++i) {
-                if (keyString == mCache[i].first) {
-                    retVal = i;
-                }
-            }
-            return retVal;
-        }
-
-        // pops plan from the back of cache(queue)
-        void popPlan() {
-            if (!mCache.empty()) {
-                // destroy the clfft plan associated with the
-                // least recently used plan
-                CLFFT_CHECK(clfftDestroyPlan(&mCache.back().second));
-                // now pop the entry from cache
-                mCache.pop_back();
-            }
-        }
-
-        // pushes plan to the front of cache(queue)
-        void pushPlan(std::string keyString, clfftPlanHandle plan) {
-            if (mCache.size()>mMaxCacheSize) {
-                popPlan();
-            }
-            mCache.push_front(FFTPlanPair(keyString, plan));
-        }
-
-    private:
-        clFFTPlanner() : mMaxCacheSize(5) {
-            CLFFT_CHECK(clfftInitSetupData(&mFFTSetup));
-            CLFFT_CHECK(clfftSetup(&mFFTSetup));
-        }
-        clFFTPlanner(clFFTPlanner const&);
-        void operator=(clFFTPlanner const&);
-
-        clfftSetupData  mFFTSetup;
-
-        size_t       mMaxCacheSize;
-        FFTPlanCache mCache;
-};
-
-void find_clfft_plan(clfftPlanHandle &plan,
-                     clfftLayout iLayout, clfftLayout oLayout,
-                     clfftDim rank, size_t *clLengths,
-                     size_t *istrides, size_t idist,
-                     size_t *ostrides, size_t odist,
-                     clfftPrecision precision, size_t batch)
-{
-    // create the key string
-    char key_str_temp[64];
-    sprintf(key_str_temp, "%d:%d:%d:", iLayout, oLayout, rank);
-
-    string key_string(key_str_temp);
-
-    /* WARNING: DO NOT CHANGE sprintf format specifier */
-    for(int r=0; r<rank; ++r) {
-        sprintf(key_str_temp, SIZE_T_FRMT_SPECIFIER ":", clLengths[r]);
-        key_string.append(std::string(key_str_temp));
-    }
-
-    if(istrides!=NULL) {
-        for(int r=0; r<rank; ++r) {
-            sprintf(key_str_temp, SIZE_T_FRMT_SPECIFIER ":", istrides[r]);
-            key_string.append(std::string(key_str_temp));
-        }
-        sprintf(key_str_temp, SIZE_T_FRMT_SPECIFIER ":", idist);
-        key_string.append(std::string(key_str_temp));
-    }
-
-    if (ostrides!=NULL) {
-        for(int r=0; r<rank; ++r) {
-            sprintf(key_str_temp, SIZE_T_FRMT_SPECIFIER ":", ostrides[r]);
-            key_string.append(std::string(key_str_temp));
-        }
-        sprintf(key_str_temp, SIZE_T_FRMT_SPECIFIER ":", odist);
-        key_string.append(std::string(key_str_temp));
-    }
-
-    sprintf(key_str_temp, "%d:" SIZE_T_FRMT_SPECIFIER, (int)precision, batch);
-    key_string.append(std::string(key_str_temp));
-
-    // find the matching plan_index in the array clFFTPlanner::mKeys
-    clFFTPlanner &planner = clFFTPlanner::getInstance();
-
-    int planIndex = planner.findIfPlanExists(key_string);
-
-    // if found a valid plan, return it
-    if (planIndex!=-1) {
-        plan = planner.getPlan(planIndex);
-        return;
-    }
-
-    clfftPlanHandle temp;
-
-    // getContext() returns object of type Context
-    // Context() returns the actual cl_context handle
-    CLFFT_CHECK(clfftCreateDefaultPlan(&temp, getContext()(), rank, clLengths));
-
-    // complex to complex
-    if (iLayout == oLayout) {
-        CLFFT_CHECK(clfftSetResultLocation(temp, CLFFT_INPLACE));
-    } else {
-        CLFFT_CHECK(clfftSetResultLocation(temp, CLFFT_OUTOFPLACE));
-    }
-
-    CLFFT_CHECK(clfftSetLayout(temp, iLayout, oLayout));
-    CLFFT_CHECK(clfftSetPlanBatchSize(temp, batch));
-    CLFFT_CHECK(clfftSetPlanDistance(temp, idist, odist));
-    CLFFT_CHECK(clfftSetPlanInStride(temp, rank, istrides));
-    CLFFT_CHECK(clfftSetPlanOutStride(temp, rank, ostrides));
-    CLFFT_CHECK(clfftSetPlanPrecision(temp, precision));
-    CLFFT_CHECK(clfftSetPlanScale(temp, CLFFT_BACKWARD, 1.0));
-
-    // getQueue() returns object of type CommandQueue
-    // CommandQueue() returns the actual cl_command_queue handle
-    CLFFT_CHECK(clfftBakePlan(temp, 1, &(getQueue()()), NULL, NULL));
-
-    plan = temp;
-
-    // push the plan into plan cache
-    planner.pushPlan(key_string, plan);
-}
-
 void setFFTPlanCacheSize(size_t numPlans)
 {
-    clFFTPlanner::getInstance().setMaxCacheSize(numPlans);
+    getclfftPlanManager().setMaxCacheSize(numPlans);
 }
 
 template<typename T> struct Precision;
@@ -268,7 +85,7 @@ void fft_inplace(Array<T> &in)
         batch *= tdims[i];
     }
 
-    find_clfft_plan(plan,
+    clfft::findPlan(plan,
                     CLFFT_COMPLEX_INTERLEAVED,
                     CLFFT_COMPLEX_INTERLEAVED,
                     (clfftDim)rank, tdims,
@@ -309,7 +126,7 @@ Array<Tc> fft_r2c(const Array<Tr> &in)
         batch *= tdims[i];
     }
 
-    find_clfft_plan(plan,
+    clfft::findPlan(plan,
                     CLFFT_REAL,
                     CLFFT_HERMITIAN_INTERLEAVED,
                     (clfftDim)rank, tdims,
@@ -349,7 +166,7 @@ Array<Tr> fft_c2r(const Array<Tc> &in, const dim4 &odims)
         batch *= tdims[i];
     }
 
-    find_clfft_plan(plan,
+    clfft::findPlan(plan,
                     CLFFT_HERMITIAN_INTERLEAVED,
                     CLFFT_REAL,
                     (clfftDim)rank, tdims,
diff --git a/src/backend/opencl/memory.hpp b/src/backend/opencl/memory.hpp
index 549dfb3f08..823fd4581f 100644
--- a/src/backend/opencl/memory.hpp
+++ b/src/backend/opencl/memory.hpp
@@ -8,7 +8,12 @@
  ********************************************************/
 #pragma once
 
-#include <platform.hpp>
+#include <cstdlib>
+
+namespace cl
+{
+class Buffer;
+}
 
 namespace opencl
 {
diff --git a/src/backend/opencl/platform.cpp b/src/backend/opencl/platform.cpp
index c3d5cf7f16..9dcffc89cd 100644
--- a/src/backend/opencl/platform.cpp
+++ b/src/backend/opencl/platform.cpp
@@ -837,6 +837,11 @@ bool& evalFlag()
     return flag;
 }
 
+clfft::clFFTPlanner& getclfftPlanManager()
+{
+    return DeviceManager::getInstance().clfftManagers[getActiveDeviceId()];
+}
+
 }
 
 using namespace opencl;
diff --git a/src/backend/opencl/platform.hpp b/src/backend/opencl/platform.hpp
index f1ed903974..da6905c42a 100644
--- a/src/backend/opencl/platform.hpp
+++ b/src/backend/opencl/platform.hpp
@@ -22,14 +22,25 @@
 #include <CL/cl2.hpp>
 #pragma GCC diagnostic pop
 
+#include <memory>
 #include <vector>
 #include <string>
 
+#include <clfftManager.hpp>
+
 namespace opencl
 {
 
+///////////////////////// BEGIN Sub-Managers ///////////////////
+//
+clfft::clFFTPlanner& getclfftPlanManager();
+//
+///////////////////////// END Sub-Managers /////////////////////
+
 class DeviceManager
 {
+    friend clfft::clFFTPlanner& getclfftPlanManager();
+
     friend std::string getDeviceInfo();
 
     friend int getDeviceCount();
@@ -97,6 +108,8 @@ class DeviceManager
 
         unsigned mActiveCtxId;
         unsigned mActiveQId;
+
+        clfft::clFFTPlanner clfftManagers[MAX_DEVICES];
 };
 
 int getBackend();

From d2cc3ba98d61d1b8afef6666e0a14e4b79f69c7b Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Tue, 10 Jan 2017 10:35:39 +0530
Subject: [PATCH 14/82] Memory-managers(opencl) initialize from DeviceManager

---
 src/backend/opencl/err_opencl.hpp    |   2 +-
 src/backend/opencl/memory.cpp        | 142 +--------------------------
 src/backend/opencl/memoryManager.cpp |  86 ++++++++++++++++
 src/backend/opencl/memoryManager.hpp |  68 +++++++++++++
 src/backend/opencl/platform.cpp      |  45 +++++++--
 src/backend/opencl/platform.hpp      |  15 +++
 6 files changed, 209 insertions(+), 149 deletions(-)
 create mode 100644 src/backend/opencl/memoryManager.cpp
 create mode 100644 src/backend/opencl/memoryManager.hpp

diff --git a/src/backend/opencl/err_opencl.hpp b/src/backend/opencl/err_opencl.hpp
index 5d12ab2db0..841670e97e 100644
--- a/src/backend/opencl/err_opencl.hpp
+++ b/src/backend/opencl/err_opencl.hpp
@@ -8,7 +8,7 @@
  ********************************************************/
 
 #pragma once
-#include <stdio.h>
+#include <cstdio>
 #include <errorcodes.hpp>
 #include <err_common.hpp>
 #include <platform.hpp>
diff --git a/src/backend/opencl/memory.cpp b/src/backend/opencl/memory.cpp
index 59f8c31154..3542e3451a 100644
--- a/src/backend/opencl/memory.cpp
+++ b/src/backend/opencl/memory.cpp
@@ -8,150 +8,14 @@
  ********************************************************/
 
 #include <memory.hpp>
-#include <dispatch.hpp>
-#include <map>
-#include <iostream>
-#include <iomanip>
-#include <string>
-#include <types.hpp>
-#include <err_opencl.hpp>
-
-#include <MemoryManager.hpp>
 
-#ifndef AF_MEM_DEBUG
-#define AF_MEM_DEBUG 0
-#endif
-
-#ifndef AF_OPENCL_MEM_DEBUG
-#define AF_OPENCL_MEM_DEBUG 0
-#endif
+#include <memoryManager.hpp>
+#include <platform.hpp>
+#include <types.hpp>
 
 namespace opencl
 {
 
-class MemoryManager  : public common::MemoryManager
-{
-    int getActiveDeviceId();
-    size_t getMaxMemorySize(int id);
-public:
-    MemoryManager();
-    void *nativeAlloc(const size_t bytes);
-    void nativeFree(void *ptr);
-    ~MemoryManager()
-    {
-        common::lock_guard_t lock(this->memory_mutex);
-        for (int n = 0; n < getDeviceCount(); n++) {
-            opencl::setDevice(n);
-            this->garbageCollect();
-        }
-    }
-};
-
-class MemoryManagerPinned  : public common::MemoryManager
-{
-    std::vector<
-        std::map<void *, cl::Buffer>
-        > pinned_maps;
-    int getActiveDeviceId();
-    size_t getMaxMemorySize(int id);
-
-public:
-
-    MemoryManagerPinned();
-
-    void *nativeAlloc(const size_t bytes);
-    void nativeFree(void *ptr);
-
-    ~MemoryManagerPinned()
-    {
-        common::lock_guard_t lock(this->memory_mutex);
-        for (int n = 0; n < getDeviceCount(); n++) {
-            opencl::setDevice(n);
-            this->garbageCollect();
-            auto pinned_curr_iter = pinned_maps[n].begin();
-            auto pinned_end_iter  = pinned_maps[n].end();
-            while (pinned_curr_iter != pinned_end_iter) {
-                pinned_maps[n].erase(pinned_curr_iter++);
-            }
-        }
-    }
-};
-
-int MemoryManager::getActiveDeviceId()
-{
-    return opencl::getActiveDeviceId();
-}
-
-size_t MemoryManager::getMaxMemorySize(int id)
-{
-    return opencl::getDeviceMemorySize(id);
-}
-
-MemoryManager::MemoryManager() :
-    common::MemoryManager(getDeviceCount(), common::MAX_BUFFERS, AF_MEM_DEBUG || AF_OPENCL_MEM_DEBUG)
-{
-    this->setMaxMemorySize();
-}
-
-void *MemoryManager::nativeAlloc(const size_t bytes)
-{
-    return (void *)(new cl::Buffer(getContext(), CL_MEM_READ_WRITE, bytes));
-}
-
-void MemoryManager::nativeFree(void *ptr)
-{
-    delete (cl::Buffer *)ptr;
-}
-
-static MemoryManager &getMemoryManager()
-{
-    static MemoryManager instance;
-    return instance;
-}
-
-int MemoryManagerPinned::getActiveDeviceId()
-{
-    return opencl::getActiveDeviceId();
-}
-
-size_t MemoryManagerPinned::getMaxMemorySize(int id)
-{
-    return opencl::getDeviceMemorySize(id);
-}
-
-MemoryManagerPinned::MemoryManagerPinned() :
-    common::MemoryManager(getDeviceCount(), common::MAX_BUFFERS, AF_MEM_DEBUG || AF_OPENCL_MEM_DEBUG),
-    pinned_maps(getDeviceCount())
-{
-    this->setMaxMemorySize();
-}
-
-void *MemoryManagerPinned::nativeAlloc(const size_t bytes)
-{
-    void *ptr = NULL;
-    cl::Buffer buf= cl::Buffer(getContext(), CL_MEM_ALLOC_HOST_PTR, bytes);
-    ptr = getQueue().enqueueMapBuffer(buf, true, CL_MAP_READ | CL_MAP_WRITE, 0, bytes);
-    pinned_maps[opencl::getActiveDeviceId()][ptr] = buf;
-    return ptr;
-}
-
-void MemoryManagerPinned::nativeFree(void *ptr)
-{
-    int n = opencl::getActiveDeviceId();
-    auto iter = pinned_maps[n].find(ptr);
-
-    if (iter != pinned_maps[n].end()) {
-        getQueue().enqueueUnmapMemObject(pinned_maps[n][ptr], ptr);
-        pinned_maps[n].erase(iter);
-    }
-}
-
-static MemoryManagerPinned &getMemoryManagerPinned()
-{
-    static MemoryManagerPinned instance;
-    return instance;
-}
-
 void setMemStepSize(size_t step_bytes)
 {
     getMemoryManager().setMemStepSize(step_bytes);
diff --git a/src/backend/opencl/memoryManager.cpp b/src/backend/opencl/memoryManager.cpp
new file mode 100644
index 0000000000..c329faef5c
--- /dev/null
+++ b/src/backend/opencl/memoryManager.cpp
@@ -0,0 +1,86 @@
+/*******************************************************
+ * Copyright (c) 2016, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <memoryManager.hpp>
+
+#ifndef AF_MEM_DEBUG
+#define AF_MEM_DEBUG 0
+#endif
+
+#ifndef AF_OPENCL_MEM_DEBUG
+#define AF_OPENCL_MEM_DEBUG 0
+#endif
+
+namespace opencl
+{
+
+int MemoryManager::getActiveDeviceId()
+{
+    return opencl::getActiveDeviceId();
+}
+
+size_t MemoryManager::getMaxMemorySize(int id)
+{
+    return opencl::getDeviceMemorySize(id);
+}
+
+MemoryManager::MemoryManager() :
+    common::MemoryManager(getDeviceCount(), common::MAX_BUFFERS, AF_MEM_DEBUG || AF_OPENCL_MEM_DEBUG)
+{
+    this->setMaxMemorySize();
+}
+
+void *MemoryManager::nativeAlloc(const size_t bytes)
+{
+    return (void *)(new cl::Buffer(getContext(), CL_MEM_READ_WRITE, bytes));
+}
+
+void MemoryManager::nativeFree(void *ptr)
+{
+    delete (cl::Buffer *)ptr;
+}
+
+int MemoryManagerPinned::getActiveDeviceId()
+{
+    return opencl::getActiveDeviceId();
+}
+
+size_t MemoryManagerPinned::getMaxMemorySize(int id)
+{
+    return opencl::getDeviceMemorySize(id);
+}
+
+MemoryManagerPinned::MemoryManagerPinned() :
+    common::MemoryManager(getDeviceCount(), common::MAX_BUFFERS, AF_MEM_DEBUG || AF_OPENCL_MEM_DEBUG),
+    pinned_maps(getDeviceCount())
+{
+    this->setMaxMemorySize();
+}
+
+void *MemoryManagerPinned::nativeAlloc(const size_t bytes)
+{
+    void *ptr = NULL;
+    cl::Buffer buf= cl::Buffer(getContext(), CL_MEM_ALLOC_HOST_PTR, bytes);
+    ptr = getQueue().enqueueMapBuffer(buf, true, CL_MAP_READ | CL_MAP_WRITE, 0, bytes);
+    pinned_maps[opencl::getActiveDeviceId()][ptr] = buf;
+    return ptr;
+}
+
+void MemoryManagerPinned::nativeFree(void *ptr)
+{
+    int n = opencl::getActiveDeviceId();
+    auto iter = pinned_maps[n].find(ptr);
+
+    if (iter != pinned_maps[n].end()) {
+        getQueue().enqueueUnmapMemObject(pinned_maps[n][ptr], ptr);
+        pinned_maps[n].erase(iter);
+    }
+}
+
+}
diff --git a/src/backend/opencl/memoryManager.hpp b/src/backend/opencl/memoryManager.hpp
new file mode 100644
index 0000000000..eab26ed235
--- /dev/null
+++ b/src/backend/opencl/memoryManager.hpp
@@ -0,0 +1,68 @@
+/*******************************************************
+ * Copyright (c) 2014, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <platform.hpp>
+#include <MemoryManager.hpp>
+
+#include <map>
+
+namespace opencl
+{
+
+class MemoryManager  : public common::MemoryManager
+{
+    int getActiveDeviceId();
+    size_t getMaxMemorySize(int id);
+public:
+    MemoryManager();
+    void *nativeAlloc(const size_t bytes);
+    void nativeFree(void *ptr);
+    ~MemoryManager()
+    {
+        common::lock_guard_t lock(this->memory_mutex);
+        for (int n = 0; n < getDeviceCount(); n++) {
+            opencl::setDevice(n);
+            this->garbageCollect();
+        }
+    }
+};
+
+class MemoryManagerPinned  : public common::MemoryManager
+{
+    std::vector<
+        std::map<void *, cl::Buffer>
+        > pinned_maps;
+    int getActiveDeviceId();
+    size_t getMaxMemorySize(int id);
+
+public:
+
+    MemoryManagerPinned();
+
+    void *nativeAlloc(const size_t bytes);
+    void nativeFree(void *ptr);
+
+    ~MemoryManagerPinned()
+    {
+        common::lock_guard_t lock(this->memory_mutex);
+        for (int n = 0; n < getDeviceCount(); n++) {
+            opencl::setDevice(n);
+            this->garbageCollect();
+            auto pinned_curr_iter = pinned_maps[n].begin();
+            auto pinned_end_iter  = pinned_maps[n].end();
+            while (pinned_curr_iter != pinned_end_iter) {
+                pinned_maps[n].erase(pinned_curr_iter++);
+            }
+        }
+    }
+};
+
+}
diff --git a/src/backend/opencl/platform.cpp b/src/backend/opencl/platform.cpp
index 9dcffc89cd..5b5ac9e985 100644
--- a/src/backend/opencl/platform.cpp
+++ b/src/backend/opencl/platform.cpp
@@ -25,23 +25,24 @@
 #include <af/version.h>
 #include <af/opencl.h>
 #include <defines.hpp>
-#include <version.hpp>
+#include <errorcodes.hpp>
+#include <err_opencl.hpp>
+#include <host_memory.hpp>
+#include <memoryManager.hpp>
 #include <platform.hpp>
 #include <util.hpp>
-#include <functional>
+#include <version.hpp>
+
 #include <algorithm>
 #include <cctype>
-#include <vector>
+#include <functional>
+#include <map>
+#include <mutex>
 #include <string>
 #include <sstream>
 #include <stdexcept>
 #include <cstring>
-#include <algorithm>
-#include <map>
-#include <errorcodes.hpp>
-#include <err_opencl.hpp>
-#include <util.hpp>
-#include <host_memory.hpp>
+#include <vector>
 
 using std::string;
 using std::vector;
@@ -837,6 +838,32 @@ bool& evalFlag()
     return flag;
 }
 
+MemoryManager& getMemoryManager()
+{
+    static std::once_flag myFlag;
+
+    DeviceManager& inst = DeviceManager::getInstance();
+
+    std::call_once(myFlag, [&]() {
+                inst.memManager.reset(new MemoryManager());
+            });
+
+    return *(inst.memManager.get());
+}
+
+MemoryManagerPinned& getMemoryManagerPinned()
+{
+    static std::once_flag myFlag;
+
+    DeviceManager& inst = DeviceManager::getInstance();
+
+    std::call_once(myFlag, [&]() {
+                inst.pinnedMemManager.reset(new MemoryManagerPinned());
+            });
+
+    return *(inst.pinnedMemManager.get());
+}
+
 clfft::clFFTPlanner& getclfftPlanManager()
 {
     return DeviceManager::getInstance().clfftManagers[getActiveDeviceId()];
diff --git a/src/backend/opencl/platform.hpp b/src/backend/opencl/platform.hpp
index da6905c42a..c921ca69b1 100644
--- a/src/backend/opencl/platform.hpp
+++ b/src/backend/opencl/platform.hpp
@@ -31,14 +31,26 @@
 namespace opencl
 {
 
+// Forward Declarations
+class MemoryManager;
+class MemoryManagerPinned;
+
 ///////////////////////// BEGIN Sub-Managers ///////////////////
 //
+MemoryManager &getMemoryManager();
+
+MemoryManagerPinned& getMemoryManagerPinned();
+
 clfft::clFFTPlanner& getclfftPlanManager();
 //
 ///////////////////////// END Sub-Managers /////////////////////
 
 class DeviceManager
 {
+    friend MemoryManager &getMemoryManager();
+
+    friend MemoryManagerPinned& getMemoryManagerPinned();
+
     friend clfft::clFFTPlanner& getclfftPlanManager();
 
     friend std::string getDeviceInfo();
@@ -109,6 +121,9 @@ class DeviceManager
         unsigned mActiveCtxId;
         unsigned mActiveQId;
 
+        std::unique_ptr<MemoryManager> memManager;
+        std::unique_ptr<MemoryManagerPinned> pinnedMemManager;
+
         clfft::clFFTPlanner clfftManagers[MAX_DEVICES];
 };
 

From a7dd2e804179c57eef62dbc1f0972884a4bd6dfa Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Tue, 10 Jan 2017 11:01:55 +0530
Subject: [PATCH 15/82] Init opencl gfx manager from DeviceManager

---
 src/backend/opencl/hist_graphics.cpp  |  8 ++++----
 src/backend/opencl/image.cpp          |  9 +++++----
 src/backend/opencl/interopManager.cpp |  8 ++------
 src/backend/opencl/interopManager.hpp |  9 +++++++--
 src/backend/opencl/platform.cpp       | 22 ++++++++++++++++++----
 src/backend/opencl/platform.hpp       |  6 ++++++
 src/backend/opencl/plot.cpp           |  8 ++++----
 src/backend/opencl/surface.cpp        |  8 ++++----
 src/backend/opencl/vector_field.cpp   |  8 ++++----
 9 files changed, 54 insertions(+), 32 deletions(-)

diff --git a/src/backend/opencl/hist_graphics.cpp b/src/backend/opencl/hist_graphics.cpp
index 50a78be927..675110e8b7 100644
--- a/src/backend/opencl/hist_graphics.cpp
+++ b/src/backend/opencl/hist_graphics.cpp
@@ -9,11 +9,11 @@
 
 #if defined (WITH_GRAPHICS)
 
-#include <interopManager.hpp>
 #include <Array.hpp>
-#include <hist_graphics.hpp>
-#include <err_opencl.hpp>
 #include <debug_opencl.hpp>
+#include <err_opencl.hpp>
+#include <hist_graphics.hpp>
+#include <interopManager.hpp>
 
 namespace opencl
 {
@@ -27,7 +27,7 @@ void copy_histogram(const Array<T> &data, const forge::Histogram* hist)
         const cl::Buffer *d_P = data.get();
         size_t bytes = hist->verticesSize();
 
-        InteropManager& intrpMngr = InteropManager::getInstance();
+        InteropManager& intrpMngr = getGfxInteropManager();
 
         cl::Buffer **resources = intrpMngr.getBufferResource(hist);
 
diff --git a/src/backend/opencl/image.cpp b/src/backend/opencl/image.cpp
index 8f403ea1b4..27258694cd 100644
--- a/src/backend/opencl/image.cpp
+++ b/src/backend/opencl/image.cpp
@@ -9,11 +9,12 @@
 
 #if defined(WITH_GRAPHICS)
 
-#include <interopManager.hpp>
 #include <Array.hpp>
-#include <image.hpp>
-#include <err_opencl.hpp>
 #include <debug_opencl.hpp>
+#include <err_opencl.hpp>
+#include <image.hpp>
+#include <interopManager.hpp>
+
 #include <stdexcept>
 #include <vector>
 
@@ -26,7 +27,7 @@ void copy_image(const Array<T> &in, const forge::Image* image)
 {
     if (isGLSharingSupported()) {
         CheckGL("Begin opencl resource copy");
-        InteropManager& intrpMngr = InteropManager::getInstance();
+        InteropManager& intrpMngr = getGfxInteropManager();
 
         cl::Buffer **resources = intrpMngr.getBufferResource(image);
         const cl::Buffer *d_X = in.get();
diff --git a/src/backend/opencl/interopManager.cpp b/src/backend/opencl/interopManager.cpp
index ff2da42efc..d8ed8641bb 100644
--- a/src/backend/opencl/interopManager.cpp
+++ b/src/backend/opencl/interopManager.cpp
@@ -11,6 +11,8 @@
 
 #include <interopManager.hpp>
 
+#include <platform.hpp>
+
 namespace opencl
 {
 
@@ -34,12 +36,6 @@ InteropManager::~InteropManager()
     }
 }
 
-InteropManager& InteropManager::getInstance()
-{
-    static InteropManager my_instance;
-    return my_instance;
-}
-
 interop_t& InteropManager::getDeviceMap(int device)
 {
     return (device == -1) ? interop_maps[getActiveDeviceId()] : interop_maps[device];
diff --git a/src/backend/opencl/interopManager.hpp b/src/backend/opencl/interopManager.hpp
index 6858f51c4b..d495794e36 100644
--- a/src/backend/opencl/interopManager.hpp
+++ b/src/backend/opencl/interopManager.hpp
@@ -14,9 +14,15 @@
 
 #include <graphics_common.hpp>
 #include <platform.hpp>
+
 #include <map>
 #include <vector>
 
+namespace cl
+{
+class Buffer;
+}
+
 namespace opencl
 {
 
@@ -29,7 +35,7 @@ class InteropManager
         interop_t interop_maps[DeviceManager::MAX_DEVICES];
 
     public:
-        static InteropManager& getInstance();
+        InteropManager() {}
         ~InteropManager();
         cl::Buffer** getBufferResource(const forge::Image        *handle);
         cl::Buffer** getBufferResource(const forge::Plot         *handle);
@@ -38,7 +44,6 @@ class InteropManager
         cl::Buffer** getBufferResource(const forge::VectorField  *handle);
 
     protected:
-        InteropManager() {}
         InteropManager(InteropManager const&);
         void operator=(InteropManager const&);
         interop_t& getDeviceMap(int device = -1); // default will return current device
diff --git a/src/backend/opencl/platform.cpp b/src/backend/opencl/platform.cpp
index 5b5ac9e985..9519e0099e 100644
--- a/src/backend/opencl/platform.cpp
+++ b/src/backend/opencl/platform.cpp
@@ -28,6 +28,7 @@
 #include <errorcodes.hpp>
 #include <err_opencl.hpp>
 #include <host_memory.hpp>
+#include <interopManager.hpp>
 #include <memoryManager.hpp>
 #include <platform.hpp>
 #include <util.hpp>
@@ -840,11 +841,11 @@ bool& evalFlag()
 
 MemoryManager& getMemoryManager()
 {
-    static std::once_flag myFlag;
+    static std::once_flag flag;
 
     DeviceManager& inst = DeviceManager::getInstance();
 
-    std::call_once(myFlag, [&]() {
+    std::call_once(flag, [&]() {
                 inst.memManager.reset(new MemoryManager());
             });
 
@@ -853,17 +854,30 @@ MemoryManager& getMemoryManager()
 
 MemoryManagerPinned& getMemoryManagerPinned()
 {
-    static std::once_flag myFlag;
+    static std::once_flag flag;
 
     DeviceManager& inst = DeviceManager::getInstance();
 
-    std::call_once(myFlag, [&]() {
+    std::call_once(flag, [&]() {
                 inst.pinnedMemManager.reset(new MemoryManagerPinned());
             });
 
     return *(inst.pinnedMemManager.get());
 }
 
+InteropManager& getGfxInteropManager()
+{
+    static std::once_flag flag;
+
+    DeviceManager& inst = DeviceManager::getInstance();
+
+    std::call_once(flag, [&]() {
+                inst.gfxManager.reset(new InteropManager());
+            });
+
+    return *(inst.gfxManager.get());
+}
+
 clfft::clFFTPlanner& getclfftPlanManager()
 {
     return DeviceManager::getInstance().clfftManagers[getActiveDeviceId()];
diff --git a/src/backend/opencl/platform.hpp b/src/backend/opencl/platform.hpp
index c921ca69b1..c0803a8deb 100644
--- a/src/backend/opencl/platform.hpp
+++ b/src/backend/opencl/platform.hpp
@@ -34,6 +34,7 @@ namespace opencl
 // Forward Declarations
 class MemoryManager;
 class MemoryManagerPinned;
+class InteropManager;
 
 ///////////////////////// BEGIN Sub-Managers ///////////////////
 //
@@ -41,6 +42,8 @@ MemoryManager &getMemoryManager();
 
 MemoryManagerPinned& getMemoryManagerPinned();
 
+InteropManager& getGfxInteropManager();
+
 clfft::clFFTPlanner& getclfftPlanManager();
 //
 ///////////////////////// END Sub-Managers /////////////////////
@@ -51,6 +54,8 @@ class DeviceManager
 
     friend MemoryManagerPinned& getMemoryManagerPinned();
 
+    friend InteropManager& getGfxInteropManager();
+
     friend clfft::clFFTPlanner& getclfftPlanManager();
 
     friend std::string getDeviceInfo();
@@ -123,6 +128,7 @@ class DeviceManager
 
         std::unique_ptr<MemoryManager> memManager;
         std::unique_ptr<MemoryManagerPinned> pinnedMemManager;
+        std::unique_ptr<InteropManager> gfxManager;
 
         clfft::clFFTPlanner clfftManagers[MAX_DEVICES];
 };
diff --git a/src/backend/opencl/plot.cpp b/src/backend/opencl/plot.cpp
index e597dff270..40240fbd10 100644
--- a/src/backend/opencl/plot.cpp
+++ b/src/backend/opencl/plot.cpp
@@ -9,12 +9,12 @@
 
 #if defined (WITH_GRAPHICS)
 
-#include <interopManager.hpp>
 #include <Array.hpp>
-#include <plot.hpp>
-#include <err_opencl.hpp>
 #include <debug_opencl.hpp>
+#include <err_opencl.hpp>
+#include <interopManager.hpp>
 #include <join.hpp>
+#include <plot.hpp>
 #include <reduce.hpp>
 #include <reorder.hpp>
 
@@ -32,7 +32,7 @@ void copy_plot(const Array<T> &P, forge::Plot* plot)
         const cl::Buffer *d_P = P.get();
         size_t bytes = plot->verticesSize();
 
-        InteropManager& intrpMngr = InteropManager::getInstance();
+        InteropManager& intrpMngr = getGfxInteropManager();
 
         cl::Buffer **resources = intrpMngr.getBufferResource(plot);
 
diff --git a/src/backend/opencl/surface.cpp b/src/backend/opencl/surface.cpp
index 8a54a8e3f6..0ed38a3e90 100644
--- a/src/backend/opencl/surface.cpp
+++ b/src/backend/opencl/surface.cpp
@@ -9,14 +9,14 @@
 
 #if defined (WITH_GRAPHICS)
 
-#include <interopManager.hpp>
 #include <Array.hpp>
-#include <surface.hpp>
-#include <err_opencl.hpp>
 #include <debug_opencl.hpp>
+#include <err_opencl.hpp>
+#include <interopManager.hpp>
 #include <join.hpp>
 #include <reduce.hpp>
 #include <reorder.hpp>
+#include <surface.hpp>
 
 using af::dim4;
 
@@ -32,7 +32,7 @@ void copy_surface(const Array<T> &P, forge::Surface* surface)
         const cl::Buffer *d_P = P.get();
         size_t bytes = surface->verticesSize();
 
-        InteropManager& intrpMngr = InteropManager::getInstance();
+        InteropManager& intrpMngr = getGfxInteropManager();
 
         cl::Buffer **resources = intrpMngr.getBufferResource(surface);
 
diff --git a/src/backend/opencl/vector_field.cpp b/src/backend/opencl/vector_field.cpp
index 5f35437e95..3fd4e32456 100644
--- a/src/backend/opencl/vector_field.cpp
+++ b/src/backend/opencl/vector_field.cpp
@@ -9,11 +9,11 @@
 
 #if defined (WITH_GRAPHICS)
 
-#include <interopManager.hpp>
 #include <Array.hpp>
-#include <vector_field.hpp>
-#include <err_opencl.hpp>
 #include <debug_opencl.hpp>
+#include <err_opencl.hpp>
+#include <interopManager.hpp>
+#include <vector_field.hpp>
 
 using af::dim4;
 
@@ -32,7 +32,7 @@ void copy_vector_field(const Array<T> &points, const Array<T> &directions,
         size_t pBytes = vector_field->verticesSize();
         size_t dBytes = vector_field->directionsSize();
 
-        InteropManager& intrpMngr = InteropManager::getInstance();
+        InteropManager& intrpMngr = getGfxInteropManager();
 
         cl::Buffer **resources = intrpMngr.getBufferResource(vector_field);
 

From 4360a34775c83548c291737e6b3f09859ba30074 Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Tue, 10 Jan 2017 11:06:42 +0530
Subject: [PATCH 16/82] Fix gfx manager initialization to be thread safe

---
 src/backend/cuda/platform.cpp | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/src/backend/cuda/platform.cpp b/src/backend/cuda/platform.cpp
index ad606b6d14..6f1f048f38 100644
--- a/src/backend/cuda/platform.cpp
+++ b/src/backend/cuda/platform.cpp
@@ -383,11 +383,11 @@ DeviceManager& DeviceManager::getInstance()
 
 MemoryManager &getMemoryManager()
 {
-    static std::once_flag myFlag;
+    static std::once_flag flag;
 
     DeviceManager& inst = DeviceManager::getInstance();
 
-    std::call_once(myFlag, [&]() {
+    std::call_once(flag, [&]() {
                 inst.memManager.reset(new cuda::MemoryManager());
             });
 
@@ -396,11 +396,11 @@ MemoryManager &getMemoryManager()
 
 MemoryManagerPinned &getMemoryManagerPinned()
 {
-    static std::once_flag myFlag;
+    static std::once_flag flag;
 
     DeviceManager& inst = DeviceManager::getInstance();
 
-    std::call_once(myFlag, [&]() {
+    std::call_once(flag, [&]() {
                 inst.pinnedMemManager.reset(new cuda::MemoryManagerPinned());
             });
 
@@ -409,10 +409,13 @@ MemoryManagerPinned &getMemoryManagerPinned()
 
 InteropManager& getGfxInteropManager()
 {
+    static std::once_flag flag;
+
     DeviceManager& inst = DeviceManager::getInstance();
 
-    if (!inst.gfxManager)
-        inst.gfxManager.reset(new cuda::InteropManager());
+    std::call_once(flag, [&]() {
+                inst.gfxManager.reset(new cuda::InteropManager());
+            });
 
     return *(inst.gfxManager.get());
 }

From ab74af83e5d936fdf6cfeedcb49e4c5a53e81f7c Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Wed, 11 Jan 2017 12:18:11 +0530
Subject: [PATCH 17/82] Remove unused link libraries from link command

---
 src/backend/cuda/CMakeLists.txt | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/backend/cuda/CMakeLists.txt b/src/backend/cuda/CMakeLists.txt
index 4da7cec1be..f5ff45092a 100644
--- a/src/backend/cuda/CMakeLists.txt
+++ b/src/backend/cuda/CMakeLists.txt
@@ -433,8 +433,7 @@ TARGET_LINK_LIBRARIES(afcuda    PRIVATE ${CUDA_CUBLAS_LIBRARIES}
                                 PRIVATE ${CUDA_cusparse_LIBRARY}
                                 PRIVATE ${CUDA_cusolver_LIBRARY}
                                 PRIVATE ${CUDA_nvvm_LIBRARY}
-                                PRIVATE ${CUDA_CUDA_LIBRARY}
-                                PRIVATE ${Boost_LIBRARIES})
+                                PRIVATE ${CUDA_CUDA_LIBRARY})
 
 LIST(LENGTH GRAPHICS_DEPENDENCIES GRAPHICS_DEPENDENCIES_LEN)
 IF(${GRAPHICS_DEPENDENCIES_LEN} GREATER 0)

From 09da66a62bfe810f6646214759ae4efa0e9ccf29 Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Wed, 11 Jan 2017 13:09:44 +0530
Subject: [PATCH 18/82] Remove unused directory from src/backend

---
 src/backend/template/.gitignore | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 delete mode 100644 src/backend/template/.gitignore

diff --git a/src/backend/template/.gitignore b/src/backend/template/.gitignore
deleted file mode 100644
index e69de29bb2..0000000000

From db3b747b692e2b8ccbb970226807751c5d751462 Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Thu, 12 Jan 2017 20:56:00 +0530
Subject: [PATCH 19/82] Abstract InteropManager(cuda/opencl) logic using CRTP

---
 src/api/c/graphics_common.cpp                 |  10 +-
 src/api/c/graphics_common.hpp                 |   2 +
 src/backend/common/InteropManager.hpp         | 123 ++++++++++++++++++
 src/backend/cuda/GraphicsResourceManager.hpp  |  57 ++++++++
 src/backend/cuda/hist_graphics.cpp            |   4 +-
 src/backend/cuda/image.cpp                    |   4 +-
 src/backend/cuda/platform.cpp                 |  15 +--
 src/backend/cuda/platform.hpp                 |  12 +-
 src/backend/cuda/plot.cpp                     |   4 +-
 src/backend/cuda/surface.cpp                  |   4 +-
 src/backend/cuda/vector_field.cpp             |   4 +-
 .../opencl/GraphicsResourceManager.cpp        |  33 +++++
 .../opencl/GraphicsResourceManager.hpp        |  35 +++++
 src/backend/opencl/hist_graphics.cpp          |   4 +-
 src/backend/opencl/image.cpp                  |   4 +-
 src/backend/opencl/interopManager.cpp         | 123 ------------------
 src/backend/opencl/interopManager.hpp         |  55 --------
 src/backend/opencl/platform.cpp               |  15 +--
 src/backend/opencl/platform.hpp               | 111 ++++++++--------
 src/backend/opencl/plot.cpp                   |   4 +-
 src/backend/opencl/surface.cpp                |   4 +-
 src/backend/opencl/vector_field.cpp           |   4 +-
 22 files changed, 348 insertions(+), 283 deletions(-)
 create mode 100644 src/backend/common/InteropManager.hpp
 create mode 100644 src/backend/cuda/GraphicsResourceManager.hpp
 create mode 100644 src/backend/opencl/GraphicsResourceManager.cpp
 create mode 100644 src/backend/opencl/GraphicsResourceManager.hpp
 delete mode 100644 src/backend/opencl/interopManager.cpp
 delete mode 100644 src/backend/opencl/interopManager.hpp

diff --git a/src/api/c/graphics_common.cpp b/src/api/c/graphics_common.cpp
index 725aac19a8..b35d15b879 100644
--- a/src/api/c/graphics_common.cpp
+++ b/src/api/c/graphics_common.cpp
@@ -10,14 +10,12 @@
 #if defined(WITH_GRAPHICS)
 
 #include <graphics_common.hpp>
+#include <glbinding/Meta.h>
 #include <err_common.hpp>
 #include <backend.hpp>
 #include <platform.hpp>
 #include <util.hpp>
 
-#include <glbinding/gl/gl.h>
-#include <glbinding/Meta.h>
-
 using namespace std;
 using namespace gl;
 
@@ -54,7 +52,7 @@ INSTANTIATE_GET_FG_TYPE(short           , forge::s16);
 gl::GLenum glErrorSkip(const char *msg, const char* file, int line)
 {
 #ifndef NDEBUG
-    gl::GLenum x = glGetError();
+    gl::GLenum x = gl::glGetError();
     if (x != GL_NO_ERROR) {
         char buf[1024];
         sprintf(buf, "GL Error Skipped at: %s:%d Message: %s Error Code: %d \"%s\"\n", file, line, msg, (int)x, glbinding::Meta::getString(x).c_str());
@@ -70,7 +68,7 @@ gl::GLenum glErrorCheck(const char *msg, const char* file, int line)
 {
 // Skipped in release mode
 #ifndef NDEBUG
-    gl::GLenum x = glGetError();
+    gl::GLenum x = gl::glGetError();
 
     if (x != GL_NO_ERROR) {
         char buf[1024];
@@ -85,7 +83,7 @@ gl::GLenum glErrorCheck(const char *msg, const char* file, int line)
 
 gl::GLenum glForceErrorCheck(const char *msg, const char* file, int line)
 {
-    gl::GLenum x = glGetError();
+    gl::GLenum x = gl::glGetError();
 
     if (x != GL_NO_ERROR) {
         char buf[1024];
diff --git a/src/api/c/graphics_common.hpp b/src/api/c/graphics_common.hpp
index 06fb68d96b..ac5ebe7709 100644
--- a/src/api/c/graphics_common.hpp
+++ b/src/api/c/graphics_common.hpp
@@ -12,9 +12,11 @@
 #if defined(WITH_GRAPHICS)
 
 #include <af/graphics.h>
+
 #include <forge.h>
 #include <glbinding/gl/gl.h>
 #include <glbinding/Binding.h>
+
 #include <vector>
 #include <map>
 #include <utility>
diff --git a/src/backend/common/InteropManager.hpp b/src/backend/common/InteropManager.hpp
new file mode 100644
index 0000000000..f32a952e27
--- /dev/null
+++ b/src/backend/common/InteropManager.hpp
@@ -0,0 +1,123 @@
+/*******************************************************
+ * Copyright (c) 2016, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#if defined(WITH_GRAPHICS)
+#include <forge.h>
+#include <err_common.hpp>
+#include <util.hpp>
+#include <cstdio>
+#include <map>
+#include <vector>
+
+namespace common
+{
+template<typename T, typename R>
+class InteropManager
+{
+    public:
+        InteropManager() {}
+
+        ~InteropManager() {
+            try {
+                destroyResources();
+            } catch (AfError &ex) {
+
+                std::string perr = getEnvVar("AF_PRINT_ERRORS");
+                if(!perr.empty()) {
+                    if(perr != "0") fprintf(stderr, "%s\n", ex.what());
+                }
+            }
+        }
+
+        R* getBufferResource(const forge::Image* image) {
+            void * key = (void*)image;
+
+            if (interopMap.find(key) == interopMap.end()) {
+                std::vector<uint32_t> handles;
+                handles.push_back(image->pixels());
+                std::vector<R> output = static_cast<T*>(this)->registerResources(handles);
+                interopMap[key] = output;
+            }
+
+            return &interopMap[key].front();
+        }
+
+        R* getBufferResource(const forge::Plot* plot) {
+            void * key = (void*)plot;
+
+            if (interopMap.find(key) == interopMap.end()) {
+                std::vector<uint32_t> handles;
+                handles.push_back(plot->vertices());
+                std::vector<R> output = static_cast<T*>(this)->registerResources(handles);
+                interopMap[key] = output;
+            }
+
+            return &interopMap[key].front();
+        }
+
+        R* getBufferResource(const forge::Histogram* histogram) {
+            void * key = (void*)histogram;
+
+            if (interopMap.find(key) == interopMap.end()) {
+                std::vector<uint32_t> handles;
+                handles.push_back(histogram->vertices());
+                std::vector<R> output = static_cast<T*>(this)->registerResources(handles);
+                interopMap[key] = output;
+            }
+
+            return &interopMap[key].front();
+        }
+
+        R* getBufferResource(const forge::Surface* surface) {
+            void * key = (void*)surface;
+
+            if (interopMap.find(key) == interopMap.end()) {
+                std::vector<uint32_t> handles;
+                handles.push_back(surface->vertices());
+                std::vector<R> output = static_cast<T*>(this)->registerResources(handles);
+                interopMap[key] = output;
+            }
+
+            return &interopMap[key].front();
+        }
+
+        R* getBufferResource(const forge::VectorField* field) {
+            void * key = (void*)field;
+
+            if (interopMap.find(key) == interopMap.end()) {
+                std::vector<uint32_t> handles;
+                handles.push_back(field->vertices());
+                handles.push_back(field->directions());
+                std::vector<R> output = static_cast<T*>(this)->registerResources(handles);
+                interopMap[key] = output;
+            }
+
+            return &interopMap[key].front();
+        }
+
+    protected:
+        InteropManager(InteropManager const&);
+        void operator=(InteropManager const&);
+
+        void destroyResources() {
+            for(auto iter : interopMap) {
+                for(auto ct : iter.second) {
+                    static_cast<T*>(this)->unregisterResource(ct);
+                }
+                iter.second.clear();
+            }
+        }
+
+    private:
+        std::map<void *, std::vector<R> > interopMap;
+};
+}
+#endif
diff --git a/src/backend/cuda/GraphicsResourceManager.hpp b/src/backend/cuda/GraphicsResourceManager.hpp
new file mode 100644
index 0000000000..b897d500c6
--- /dev/null
+++ b/src/backend/cuda/GraphicsResourceManager.hpp
@@ -0,0 +1,57 @@
+/*******************************************************
+ * Copyright (c) 2016, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#if defined(WITH_GRAPHICS)
+#if defined(OS_WIN)
+#include <windows.h>
+#endif
+
+#include <platform.hpp>
+// cuda_gl_interop.h does not include OpenGL headers for ARM
+#include <graphics_common.hpp>
+using namespace gl;
+#define GL_VERSION gl::GL_VERSION
+#define __gl_h_ //Hack to avoid gl.h inclusion by cuda_gl_interop.h
+#include <cuda_gl_interop.h>
+#include <common/InteropManager.hpp>
+#include <err_cuda.hpp>
+#include <map>
+#include <vector>
+
+namespace cuda
+{
+typedef cudaGraphicsResource_t CGR_t;
+
+class GraphicsResourceManager : public common::InteropManager<GraphicsResourceManager, CGR_t>
+{
+    public:
+        GraphicsResourceManager() {}
+
+        std::vector<CGR_t> registerResources(std::vector<uint32_t> resources) {
+            std::vector<CGR_t> output;
+
+            for (auto id: resources) {
+                CGR_t r;
+                CUDA_CHECK(cudaGraphicsGLRegisterBuffer(&r, id, cudaGraphicsMapFlagsWriteDiscard));
+                output.push_back(r);
+            }
+
+            return output;
+        }
+
+        void unregisterResource(CGR_t handle) {
+            CUDA_CHECK(cudaGraphicsUnregisterResource(handle));
+        }
+
+    protected:
+        GraphicsResourceManager(GraphicsResourceManager const&);
+        void operator=(GraphicsResourceManager const&);
+};
+}
+#endif
diff --git a/src/backend/cuda/hist_graphics.cpp b/src/backend/cuda/hist_graphics.cpp
index 750dafae3e..52c959b4d4 100644
--- a/src/backend/cuda/hist_graphics.cpp
+++ b/src/backend/cuda/hist_graphics.cpp
@@ -13,7 +13,7 @@
 #include <hist_graphics.hpp>
 #include <err_cuda.hpp>
 #include <debug_cuda.hpp>
-#include <interopManager.hpp>
+#include <common/InteropManager.hpp>
 
 namespace cuda
 {
@@ -25,7 +25,7 @@ void copy_histogram(const Array<T> &data, const forge::Histogram* hist)
     if(DeviceManager::checkGraphicsInteropCapability()) {
         const T *d_P = data.get();
 
-        InteropManager& intrpMngr = getGfxInteropManager();
+        GraphicsManager& intrpMngr = interopManager();
 
         cudaGraphicsResource_t *resources = intrpMngr.getBufferResource(hist);
         // Map resource. Copy data to VBO. Unmap resource.
diff --git a/src/backend/cuda/image.cpp b/src/backend/cuda/image.cpp
index 0492c9c6d4..816cc2b8cd 100644
--- a/src/backend/cuda/image.cpp
+++ b/src/backend/cuda/image.cpp
@@ -16,7 +16,7 @@
 #include <image.hpp>
 #include <err_cuda.hpp>
 #include <debug_cuda.hpp>
-#include <interopManager.hpp>
+#include <common/InteropManager.hpp>
 
 using af::dim4;
 
@@ -28,7 +28,7 @@ template<typename T>
 void copy_image(const Array<T> &in, const forge::Image* image)
 {
     if(DeviceManager::checkGraphicsInteropCapability()) {
-        InteropManager& intrpMngr = getGfxInteropManager();
+        GraphicsManager& intrpMngr = interopManager();
 
         cudaGraphicsResource_t *resources = intrpMngr.getBufferResource(image);
 
diff --git a/src/backend/cuda/platform.cpp b/src/backend/cuda/platform.cpp
index 6f1f048f38..447a3d03c9 100644
--- a/src/backend/cuda/platform.cpp
+++ b/src/backend/cuda/platform.cpp
@@ -25,7 +25,7 @@
 #include <util.hpp>
 #include <host_memory.hpp>
 #include <memoryManager.hpp>
-#include <interopManager.hpp>
+#include <common/InteropManager.hpp>
 
 using namespace std;
 
@@ -407,17 +407,16 @@ MemoryManagerPinned &getMemoryManagerPinned()
     return *(inst.pinnedMemManager.get());
 }
 
-InteropManager& getGfxInteropManager()
+GraphicsManager& interopManager()
 {
-    static std::once_flag flag;
-
     DeviceManager& inst = DeviceManager::getInstance();
 
-    std::call_once(flag, [&]() {
-                inst.gfxManager.reset(new cuda::InteropManager());
-            });
+    int id = cuda::getActiveDeviceId();
+
+    if (! inst.gfxManagers[id] )
+        inst.gfxManagers[id].reset(new GraphicsManager());
 
-    return *(inst.gfxManager.get());
+    return *(inst.gfxManagers[id].get());
 }
 
 cufft::cuFFTPlanner& getcufftPlanManager()
diff --git a/src/backend/cuda/platform.hpp b/src/backend/cuda/platform.hpp
index 7e256bc8eb..03ab23f5b1 100644
--- a/src/backend/cuda/platform.hpp
+++ b/src/backend/cuda/platform.hpp
@@ -14,10 +14,8 @@
 #include <memory>
 #include <vector>
 #include <string>
-#if defined(WITH_GRAPHICS)
-#include <fg/window.h>
-#endif
 
+#include <GraphicsResourceManager.hpp>
 #include <cufftManager.hpp>
 #include <cublasManager.hpp>
 #include <cusolverDnManager.hpp>
@@ -74,14 +72,14 @@ bool& evalFlag();
 
 class MemoryManager;
 class MemoryManagerPinned;
-class InteropManager;
 ///////////////////////// BEGIN Sub-Managers ///////////////////
 //
 MemoryManager& getMemoryManager();
 
 MemoryManagerPinned& getMemoryManagerPinned();
 
-InteropManager& getGfxInteropManager();
+typedef common::InteropManager<GraphicsResourceManager, CGR_t> GraphicsManager;
+GraphicsManager& interopManager();
 
 cufft::cuFFTPlanner& getcufftPlanManager();
 
@@ -106,7 +104,7 @@ class DeviceManager
 
         friend MemoryManagerPinned& getMemoryManagerPinned();
 
-        friend InteropManager& getGfxInteropManager();
+        friend GraphicsManager& interopManager();
 
         friend cufft::cuFFTPlanner& getcufftPlanManager();
 
@@ -173,7 +171,7 @@ class DeviceManager
 
         std::unique_ptr<MemoryManagerPinned> pinnedMemManager;
 
-        std::unique_ptr<InteropManager> gfxManager;
+        std::unique_ptr<GraphicsManager> gfxManagers[MAX_DEVICES];
 
         cufft::cuFFTPlanner cufftManagers[MAX_DEVICES];
 
diff --git a/src/backend/cuda/plot.cpp b/src/backend/cuda/plot.cpp
index 437c276226..f2ca1f19cf 100644
--- a/src/backend/cuda/plot.cpp
+++ b/src/backend/cuda/plot.cpp
@@ -16,7 +16,7 @@
 #include <join.hpp>
 #include <reduce.hpp>
 #include <reorder.hpp>
-#include <interopManager.hpp>
+#include <common/InteropManager.hpp>
 
 using af::dim4;
 
@@ -30,7 +30,7 @@ void copy_plot(const Array<T> &P, forge::Plot* plot)
     if(DeviceManager::checkGraphicsInteropCapability()) {
         const T *d_P = P.get();
 
-        InteropManager& intrpMngr = getGfxInteropManager();
+        GraphicsManager& intrpMngr = interopManager();
 
         cudaGraphicsResource_t *resources = intrpMngr.getBufferResource(plot);
         // Map resource. Copy data to VBO. Unmap resource.
diff --git a/src/backend/cuda/surface.cpp b/src/backend/cuda/surface.cpp
index 749ff859d6..0752a950eb 100644
--- a/src/backend/cuda/surface.cpp
+++ b/src/backend/cuda/surface.cpp
@@ -16,7 +16,7 @@
 #include <join.hpp>
 #include <reduce.hpp>
 #include <reorder.hpp>
-#include <interopManager.hpp>
+#include <common/InteropManager.hpp>
 
 using af::dim4;
 
@@ -30,7 +30,7 @@ void copy_surface(const Array<T> &P, forge::Surface* surface)
     if(DeviceManager::checkGraphicsInteropCapability()) {
         const T *d_P = P.get();
 
-        InteropManager& intrpMngr = getGfxInteropManager();
+        GraphicsManager& intrpMngr = interopManager();
 
         cudaGraphicsResource_t *resources = intrpMngr.getBufferResource(surface);
         // Map resource. Copy data to VBO. Unmap resource.
diff --git a/src/backend/cuda/vector_field.cpp b/src/backend/cuda/vector_field.cpp
index aab3f13036..36021aa1c3 100644
--- a/src/backend/cuda/vector_field.cpp
+++ b/src/backend/cuda/vector_field.cpp
@@ -13,7 +13,7 @@
 #include <vector_field.hpp>
 #include <err_cuda.hpp>
 #include <debug_cuda.hpp>
-#include <interopManager.hpp>
+#include <common/InteropManager.hpp>
 
 using af::dim4;
 
@@ -26,7 +26,7 @@ void copy_vector_field(const Array<T> &points, const Array<T> &directions,
                        forge::VectorField* vector_field)
 {
     if(DeviceManager::checkGraphicsInteropCapability()) {
-        InteropManager& intrpMngr = getGfxInteropManager();
+        GraphicsManager& intrpMngr = interopManager();
 
         cudaGraphicsResource_t *resources = intrpMngr.getBufferResource(vector_field);
 
diff --git a/src/backend/opencl/GraphicsResourceManager.cpp b/src/backend/opencl/GraphicsResourceManager.cpp
new file mode 100644
index 0000000000..8e043313ed
--- /dev/null
+++ b/src/backend/opencl/GraphicsResourceManager.cpp
@@ -0,0 +1,33 @@
+/*******************************************************
+ * Copyright (c) 2016, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#if defined(WITH_GRAPHICS)
+#include <platform.hpp>
+#include <GraphicsResourceManager.hpp>
+
+namespace opencl
+{
+std::vector<CGR_t> GraphicsResourceManager::registerResources(std::vector<uint32_t> resources)
+{
+    std::vector<CGR_t> output;
+
+    for (auto id: resources) {
+        CGR_t r = new cl::BufferGL(opencl::getContext(), CL_MEM_WRITE_ONLY, id, NULL);
+        output.push_back(r);
+    }
+
+    return output;
+}
+
+void GraphicsResourceManager::unregisterResource(CGR_t handle)
+{
+    delete handle;
+}
+}
+#endif
diff --git a/src/backend/opencl/GraphicsResourceManager.hpp b/src/backend/opencl/GraphicsResourceManager.hpp
new file mode 100644
index 0000000000..774cb0f124
--- /dev/null
+++ b/src/backend/opencl/GraphicsResourceManager.hpp
@@ -0,0 +1,35 @@
+/*******************************************************
+ * Copyright (c) 2016, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#if defined(WITH_GRAPHICS)
+#include <platform.hpp>
+#include <common/InteropManager.hpp>
+#include <map>
+#include <vector>
+
+namespace opencl
+{
+typedef cl::Buffer* CGR_t;
+
+class GraphicsResourceManager : public common::InteropManager<GraphicsResourceManager, CGR_t>
+{
+    public:
+        GraphicsResourceManager() {}
+
+        std::vector<CGR_t> registerResources(std::vector<uint32_t> resources);
+        void unregisterResource(CGR_t handle);
+
+    protected:
+        GraphicsResourceManager(GraphicsResourceManager const&);
+        void operator=(GraphicsResourceManager const&);
+};
+}
+#endif
diff --git a/src/backend/opencl/hist_graphics.cpp b/src/backend/opencl/hist_graphics.cpp
index 675110e8b7..41d7e2734c 100644
--- a/src/backend/opencl/hist_graphics.cpp
+++ b/src/backend/opencl/hist_graphics.cpp
@@ -13,7 +13,7 @@
 #include <debug_opencl.hpp>
 #include <err_opencl.hpp>
 #include <hist_graphics.hpp>
-#include <interopManager.hpp>
+#include <common/InteropManager.hpp>
 
 namespace opencl
 {
@@ -27,7 +27,7 @@ void copy_histogram(const Array<T> &data, const forge::Histogram* hist)
         const cl::Buffer *d_P = data.get();
         size_t bytes = hist->verticesSize();
 
-        InteropManager& intrpMngr = getGfxInteropManager();
+        GraphicsManager& intrpMngr = interopManager();
 
         cl::Buffer **resources = intrpMngr.getBufferResource(hist);
 
diff --git a/src/backend/opencl/image.cpp b/src/backend/opencl/image.cpp
index 27258694cd..8429e1c71d 100644
--- a/src/backend/opencl/image.cpp
+++ b/src/backend/opencl/image.cpp
@@ -13,7 +13,7 @@
 #include <debug_opencl.hpp>
 #include <err_opencl.hpp>
 #include <image.hpp>
-#include <interopManager.hpp>
+#include <common/InteropManager.hpp>
 
 #include <stdexcept>
 #include <vector>
@@ -27,7 +27,7 @@ void copy_image(const Array<T> &in, const forge::Image* image)
 {
     if (isGLSharingSupported()) {
         CheckGL("Begin opencl resource copy");
-        InteropManager& intrpMngr = getGfxInteropManager();
+        GraphicsManager& intrpMngr = interopManager();
 
         cl::Buffer **resources = intrpMngr.getBufferResource(image);
         const cl::Buffer *d_X = in.get();
diff --git a/src/backend/opencl/interopManager.cpp b/src/backend/opencl/interopManager.cpp
deleted file mode 100644
index d8ed8641bb..0000000000
--- a/src/backend/opencl/interopManager.cpp
+++ /dev/null
@@ -1,123 +0,0 @@
-/*******************************************************
- * Copyright (c) 2014, ArrayFire
- * All rights reserved.
- *
- * This file is distributed under 3-clause BSD license.
- * The complete license agreement can be obtained at:
- * http://arrayfire.com/licenses/BSD-3-Clause
- ********************************************************/
-
-#if defined(WITH_GRAPHICS)
-
-#include <interopManager.hpp>
-
-#include <platform.hpp>
-
-namespace opencl
-{
-
-void InteropManager::destroyResources()
-{
-    typedef std::vector<cl::Buffer *>::iterator buffer_t;
-    int n = getActiveDeviceId();
-    for(iter_t iter = interop_maps[n].begin(); iter != interop_maps[n].end(); iter++) {
-        for(buffer_t bt = (iter->second).begin(); bt != (iter->second).end(); bt++) {
-            delete *bt;
-        }
-        (iter->second).clear();
-    }
-}
-
-InteropManager::~InteropManager()
-{
-    for(int i = 0; i < getDeviceCount(); i++) {
-        setDevice(i);
-        destroyResources();
-    }
-}
-
-interop_t& InteropManager::getDeviceMap(int device)
-{
-    return (device == -1) ? interop_maps[getActiveDeviceId()] : interop_maps[device];
-}
-
-cl::Buffer** InteropManager::getBufferResource(const forge::Image* image)
-{
-    void * key = (void*)image;
-    interop_t& i_map = getDeviceMap();
-    iter_t iter = i_map.find(key);
-
-    if (iter == i_map.end()) {
-        std::vector<cl::Buffer *> vec(1);
-        vec[0] = new cl::BufferGL(getContext(), CL_MEM_WRITE_ONLY, image->pixels(), NULL);
-        i_map[key] = vec;
-    }
-
-    return &i_map[key].front();
-}
-
-cl::Buffer** InteropManager::getBufferResource(const forge::Plot* plot)
-{
-    void * key = (void*)plot;
-    interop_t& i_map = getDeviceMap();
-    iter_t iter = i_map.find(key);
-
-    if (iter == i_map.end()) {
-        std::vector<cl::Buffer *> vec(1);
-        vec[0] = new cl::BufferGL(getContext(), CL_MEM_WRITE_ONLY, plot->vertices(), NULL);
-        i_map[key] = vec;
-    }
-
-    return &i_map[key].front();
-}
-
-cl::Buffer** InteropManager::getBufferResource(const forge::Histogram* hist)
-{
-    void * key = (void*)hist;
-    interop_t& i_map = getDeviceMap();
-    iter_t iter = i_map.find(key);
-
-    if (iter == i_map.end()) {
-        std::vector<cl::Buffer *> vec(1);
-        vec[0] = new cl::BufferGL(getContext(), CL_MEM_WRITE_ONLY, hist->vertices(), NULL);
-        i_map[key] = vec;
-    }
-
-    return &i_map[key].front();
-}
-
-cl::Buffer** InteropManager::getBufferResource(const forge::Surface* surface)
-{
-    void * key = (void*)surface;
-    interop_t& i_map = getDeviceMap();
-    iter_t iter = i_map.find(key);
-
-    if (iter == i_map.end()) {
-        std::vector<cl::Buffer *> vec(1);
-        vec[0] = new cl::BufferGL(getContext(), CL_MEM_WRITE_ONLY, surface->vertices(), NULL);
-        i_map[key] = vec;
-    }
-
-    return &i_map[key].front();
-}
-
-cl::Buffer** InteropManager::getBufferResource(const forge::VectorField* vector_field)
-{
-    void * key = (void*)vector_field;
-    interop_t& i_map = getDeviceMap();
-    iter_t iter = i_map.find(key);
-
-    if (iter == i_map.end()) {
-        std::vector<cl::Buffer *> vec(2);
-        vec[0] = new cl::BufferGL(getContext(), CL_MEM_WRITE_ONLY, vector_field->vertices(), NULL);
-        vec[1] = new cl::BufferGL(getContext(), CL_MEM_WRITE_ONLY, vector_field->directions(), NULL);
-        i_map[key] = vec;
-    }
-
-    return &i_map[key].front();
-}
-
-}
-
-#endif
-
diff --git a/src/backend/opencl/interopManager.hpp b/src/backend/opencl/interopManager.hpp
deleted file mode 100644
index d495794e36..0000000000
--- a/src/backend/opencl/interopManager.hpp
+++ /dev/null
@@ -1,55 +0,0 @@
-/*******************************************************
- * Copyright (c) 2014, ArrayFire
- * All rights reserved.
- *
- * This file is distributed under 3-clause BSD license.
- * The complete license agreement can be obtained at:
- * http://arrayfire.com/licenses/BSD-3-Clause
- ********************************************************/
-
-#if defined(WITH_GRAPHICS)
-
-#include <glbinding/gl/gl.h>
-#include <glbinding/Binding.h>
-
-#include <graphics_common.hpp>
-#include <platform.hpp>
-
-#include <map>
-#include <vector>
-
-namespace cl
-{
-class Buffer;
-}
-
-namespace opencl
-{
-
-typedef std::map<void *, std::vector<cl::Buffer*> > interop_t;
-typedef interop_t::iterator iter_t;
-
-class InteropManager
-{
-    private:
-        interop_t interop_maps[DeviceManager::MAX_DEVICES];
-
-    public:
-        InteropManager() {}
-        ~InteropManager();
-        cl::Buffer** getBufferResource(const forge::Image        *handle);
-        cl::Buffer** getBufferResource(const forge::Plot         *handle);
-        cl::Buffer** getBufferResource(const forge::Histogram    *handle);
-        cl::Buffer** getBufferResource(const forge::Surface      *handle);
-        cl::Buffer** getBufferResource(const forge::VectorField  *handle);
-
-    protected:
-        InteropManager(InteropManager const&);
-        void operator=(InteropManager const&);
-        interop_t& getDeviceMap(int device = -1); // default will return current device
-        void destroyResources();
-};
-
-}
-
-#endif
diff --git a/src/backend/opencl/platform.cpp b/src/backend/opencl/platform.cpp
index 9519e0099e..4c263e57ad 100644
--- a/src/backend/opencl/platform.cpp
+++ b/src/backend/opencl/platform.cpp
@@ -28,7 +28,7 @@
 #include <errorcodes.hpp>
 #include <err_opencl.hpp>
 #include <host_memory.hpp>
-#include <interopManager.hpp>
+#include <common/InteropManager.hpp>
 #include <memoryManager.hpp>
 #include <platform.hpp>
 #include <util.hpp>
@@ -865,17 +865,16 @@ MemoryManagerPinned& getMemoryManagerPinned()
     return *(inst.pinnedMemManager.get());
 }
 
-InteropManager& getGfxInteropManager()
+GraphicsManager& interopManager()
 {
-    static std::once_flag flag;
-
     DeviceManager& inst = DeviceManager::getInstance();
 
-    std::call_once(flag, [&]() {
-                inst.gfxManager.reset(new InteropManager());
-            });
+    int id = getActiveDeviceId();
+
+    if (! inst.gfxManagers[id] )
+        inst.gfxManagers[id].reset(new GraphicsManager());
 
-    return *(inst.gfxManager.get());
+    return *(inst.gfxManagers[id].get());
 }
 
 clfft::clFFTPlanner& getclfftPlanManager()
diff --git a/src/backend/opencl/platform.hpp b/src/backend/opencl/platform.hpp
index c0803a8deb..9badbe8a58 100644
--- a/src/backend/opencl/platform.hpp
+++ b/src/backend/opencl/platform.hpp
@@ -26,15 +26,65 @@
 #include <vector>
 #include <string>
 
+#include <GraphicsResourceManager.hpp>
 #include <clfftManager.hpp>
 
 namespace opencl
 {
+int getBackend();
+
+std::string getDeviceInfo();
+
+int getDeviceCount();
+
+int getActiveDeviceId();
+
+unsigned getMaxJitSize();
+
+const cl::Context& getContext();
+
+cl::CommandQueue& getQueue();
+
+const cl::Device& getDevice(int id = -1);
+
+size_t getDeviceMemorySize(int device);
+
+size_t getHostMemorySize();
+
+cl_device_type getDeviceType();
+
+bool isHostUnifiedMemory(const cl::Device &device);
+
+bool OpenCLCPUOffload(bool forceOffloadOSX = true);
+
+bool isGLSharingSupported();
+
+bool isDoubleSupported(int device);
+
+void devprop(char* d_name, char* d_platform, char *d_toolkit, char* d_compute);
+
+std::string getPlatformName(const cl::Device &device);
+
+int setDevice(int device);
+
+void addDeviceContext(cl_device_id dev, cl_context cxt, cl_command_queue que);
+
+void setDeviceContext(cl_device_id dev, cl_context cxt);
+
+void removeDeviceContext(cl_device_id dev, cl_context ctx);
+
+void sync(int device);
+
+bool synchronize_calls();
+
+int getActiveDeviceType();
+int getActivePlatform();
+
+bool& evalFlag();
 
 // Forward Declarations
 class MemoryManager;
 class MemoryManagerPinned;
-class InteropManager;
 
 ///////////////////////// BEGIN Sub-Managers ///////////////////
 //
@@ -42,7 +92,8 @@ MemoryManager &getMemoryManager();
 
 MemoryManagerPinned& getMemoryManagerPinned();
 
-InteropManager& getGfxInteropManager();
+typedef common::InteropManager<GraphicsResourceManager, CGR_t> GraphicsManager;
+GraphicsManager& interopManager();
 
 clfft::clFFTPlanner& getclfftPlanManager();
 //
@@ -54,7 +105,7 @@ class DeviceManager
 
     friend MemoryManagerPinned& getMemoryManagerPinned();
 
-    friend InteropManager& getGfxInteropManager();
+    friend GraphicsManager& interopManager();
 
     friend clfft::clFFTPlanner& getclfftPlanManager();
 
@@ -128,60 +179,8 @@ class DeviceManager
 
         std::unique_ptr<MemoryManager> memManager;
         std::unique_ptr<MemoryManagerPinned> pinnedMemManager;
-        std::unique_ptr<InteropManager> gfxManager;
 
+        std::unique_ptr<GraphicsManager> gfxManagers[MAX_DEVICES];
         clfft::clFFTPlanner clfftManagers[MAX_DEVICES];
 };
-
-int getBackend();
-
-std::string getDeviceInfo();
-
-int getDeviceCount();
-
-int getActiveDeviceId();
-
-unsigned getMaxJitSize();
-
-const cl::Context& getContext();
-
-cl::CommandQueue& getQueue();
-
-const cl::Device& getDevice(int id = -1);
-
-size_t getDeviceMemorySize(int device);
-
-size_t getHostMemorySize();
-
-cl_device_type getDeviceType();
-
-bool isHostUnifiedMemory(const cl::Device &device);
-
-bool OpenCLCPUOffload(bool forceOffloadOSX = true);
-
-bool isGLSharingSupported();
-
-bool isDoubleSupported(int device);
-
-void devprop(char* d_name, char* d_platform, char *d_toolkit, char* d_compute);
-
-std::string getPlatformName(const cl::Device &device);
-
-int setDevice(int device);
-
-void addDeviceContext(cl_device_id dev, cl_context cxt, cl_command_queue que);
-
-void setDeviceContext(cl_device_id dev, cl_context cxt);
-
-void removeDeviceContext(cl_device_id dev, cl_context ctx);
-
-void sync(int device);
-
-bool synchronize_calls();
-
-int getActiveDeviceType();
-int getActivePlatform();
-
-bool& evalFlag();
-
 }
diff --git a/src/backend/opencl/plot.cpp b/src/backend/opencl/plot.cpp
index 40240fbd10..a8eb7694e5 100644
--- a/src/backend/opencl/plot.cpp
+++ b/src/backend/opencl/plot.cpp
@@ -12,7 +12,7 @@
 #include <Array.hpp>
 #include <debug_opencl.hpp>
 #include <err_opencl.hpp>
-#include <interopManager.hpp>
+#include <common/InteropManager.hpp>
 #include <join.hpp>
 #include <plot.hpp>
 #include <reduce.hpp>
@@ -32,7 +32,7 @@ void copy_plot(const Array<T> &P, forge::Plot* plot)
         const cl::Buffer *d_P = P.get();
         size_t bytes = plot->verticesSize();
 
-        InteropManager& intrpMngr = getGfxInteropManager();
+        GraphicsManager& intrpMngr = interopManager();
 
         cl::Buffer **resources = intrpMngr.getBufferResource(plot);
 
diff --git a/src/backend/opencl/surface.cpp b/src/backend/opencl/surface.cpp
index 0ed38a3e90..29158de24e 100644
--- a/src/backend/opencl/surface.cpp
+++ b/src/backend/opencl/surface.cpp
@@ -12,7 +12,7 @@
 #include <Array.hpp>
 #include <debug_opencl.hpp>
 #include <err_opencl.hpp>
-#include <interopManager.hpp>
+#include <common/InteropManager.hpp>
 #include <join.hpp>
 #include <reduce.hpp>
 #include <reorder.hpp>
@@ -32,7 +32,7 @@ void copy_surface(const Array<T> &P, forge::Surface* surface)
         const cl::Buffer *d_P = P.get();
         size_t bytes = surface->verticesSize();
 
-        InteropManager& intrpMngr = getGfxInteropManager();
+        GraphicsManager& intrpMngr = interopManager();
 
         cl::Buffer **resources = intrpMngr.getBufferResource(surface);
 
diff --git a/src/backend/opencl/vector_field.cpp b/src/backend/opencl/vector_field.cpp
index 3fd4e32456..9debfc9d6a 100644
--- a/src/backend/opencl/vector_field.cpp
+++ b/src/backend/opencl/vector_field.cpp
@@ -12,7 +12,7 @@
 #include <Array.hpp>
 #include <debug_opencl.hpp>
 #include <err_opencl.hpp>
-#include <interopManager.hpp>
+#include <common/InteropManager.hpp>
 #include <vector_field.hpp>
 
 using af::dim4;
@@ -32,7 +32,7 @@ void copy_vector_field(const Array<T> &points, const Array<T> &directions,
         size_t pBytes = vector_field->verticesSize();
         size_t dBytes = vector_field->directionsSize();
 
-        InteropManager& intrpMngr = getGfxInteropManager();
+        GraphicsManager& intrpMngr = interopManager();
 
         cl::Buffer **resources = intrpMngr.getBufferResource(vector_field);
 

From c05c29b5143d8d8131beacc5ce4aa502a0462f23 Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Sat, 14 Jan 2017 09:34:58 +0530
Subject: [PATCH 20/82] Abstract cuda/opencl fft plan caching logic using CRTP

---
 src/backend/common/FFTPlanCache.hpp           |  91 +++++++++++++
 .../cuda/{cufftManager.cpp => cufft.cpp}      |  39 +++---
 src/backend/cuda/cufft.hpp                    |  59 ++++++++
 src/backend/cuda/cufftManager.hpp             | 121 -----------------
 src/backend/cuda/fft.cpp                      |  32 ++---
 src/backend/cuda/platform.cpp                 |   4 +-
 src/backend/cuda/platform.hpp                 |   9 +-
 .../opencl/{clfftManager.cpp => clfft.cpp}    |  72 ++++------
 src/backend/opencl/clfft.hpp                  |  67 +++++++++
 src/backend/opencl/clfftManager.hpp           | 127 ------------------
 src/backend/opencl/fft.cpp                    |  46 ++-----
 src/backend/opencl/platform.cpp               |   2 +-
 src/backend/opencl/platform.hpp               |   9 +-
 13 files changed, 305 insertions(+), 373 deletions(-)
 create mode 100644 src/backend/common/FFTPlanCache.hpp
 rename src/backend/cuda/{cufftManager.cpp => cufft.cpp} (77%)
 create mode 100644 src/backend/cuda/cufft.hpp
 delete mode 100644 src/backend/cuda/cufftManager.hpp
 rename src/backend/opencl/{clfftManager.cpp => clfft.cpp} (79%)
 create mode 100644 src/backend/opencl/clfft.hpp
 delete mode 100644 src/backend/opencl/clfftManager.hpp

diff --git a/src/backend/common/FFTPlanCache.hpp b/src/backend/common/FFTPlanCache.hpp
new file mode 100644
index 0000000000..bb0fa73803
--- /dev/null
+++ b/src/backend/common/FFTPlanCache.hpp
@@ -0,0 +1,91 @@
+/*******************************************************
+ * Copyright (c) 2016, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <deque>
+#include <string>
+#include <utility>
+
+namespace common
+{
+// FFTPlanCache caches backend specific fft plans
+//
+// new plan |--> IF number of plans cached is at limit, pop the least used entry and push new plan.
+//          |
+//          |--> ELSE just push the plan
+// existing plan -> reuse a plan
+template<typename T, typename P>
+class FFTPlanCache
+{
+    public:
+        FFTPlanCache() : mMaxCacheSize(5) {
+            static_cast<T*>(this)->initLibrary();
+        }
+
+        ~FFTPlanCache() {
+            static_cast<T*>(this)->deInitLibrary();
+        }
+
+        inline void maxCacheSize(size_t size) {
+            mCache.resize(size, std::make_pair<std::string, P>(std::string(""), 0));
+        }
+
+        inline size_t maxCacheSize() const {
+            return mMaxCacheSize;
+        }
+
+        inline P get(int index) const {
+            return mCache[index].second;
+        }
+
+        // iterates through plan cache from front to back
+        // of the cache(queue)
+        //
+        // A valid index of the plan in the cache is returned
+        // otherwise -1 is returned
+        int find(std::string key) const {
+            int retVal = -1;
+            for(uint i=0; i<mCache.size(); ++i) {
+                if (key == mCache[i].first) {
+                    retVal = i;
+                }
+            }
+            return retVal;
+        }
+
+        // pops plan from the back of cache(queue)
+        void pop() {
+            if (!mCache.empty()) {
+                // destroy the cufft plan associated with the
+                // least recently used plan
+                static_cast<T*>(this)->removePlan(mCache.back().second);
+                // now pop the entry from cache
+                mCache.pop_back();
+            }
+        }
+
+        // pushes plan to the front of cache(queue)
+        void push(std::string key, P plan) {
+            if (mCache.size()>mMaxCacheSize) {
+                pop();
+            }
+            mCache.push_front(std::pair<std::string, P>(key, plan));
+        }
+
+    private:
+        FFTPlanCache(FFTPlanCache const&);
+        void operator=(FFTPlanCache const&);
+
+        size_t       mMaxCacheSize;
+
+        std::deque< std::pair<std::string, P> > mCache;
+
+};
+}
diff --git a/src/backend/cuda/cufftManager.cpp b/src/backend/cuda/cufft.cpp
similarity index 77%
rename from src/backend/cuda/cufftManager.cpp
rename to src/backend/cuda/cufft.cpp
index 28c994b464..f8d36e7ee8 100644
--- a/src/backend/cuda/cufftManager.cpp
+++ b/src/backend/cuda/cufft.cpp
@@ -7,11 +7,11 @@
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
 
-#include <cuda.h> // Need this for CUDA_VERSION
 #include <platform.hpp>
+#include <cufft.hpp>
 #include <memory.hpp>
 
-namespace cufft
+namespace cuda
 {
 
 const char * _cufftGetResultString(cufftResult res)
@@ -76,10 +76,10 @@ const char * _cufftGetResultString(cufftResult res)
     return "cuFFT: unknown error";
 }
 
-void findPlan(cufftHandle &plan, int rank, int *n,
-              int *inembed, int istride, int idist,
-              int *onembed, int ostride, int odist,
-              cufftType type, int batch)
+PlanType findPlan(int rank, int *n,
+                  int *inembed, int istride, int idist,
+                  int *onembed, int ostride, int odist,
+                  cufftType type, int batch)
 {
     // create the key string
     char key_str_temp[64];
@@ -113,37 +113,34 @@ void findPlan(cufftHandle &plan, int rank, int *n,
     sprintf(key_str_temp, "%d:%d", (int)type, batch);
     key_string.append(std::string(key_str_temp));
 
-    // find the matching plan_index in the array cuFFTPlanner::mKeys
-    cuFFTPlanner &planner = cuda::getcufftPlanManager();
+    FFTManager &planner = cuda::cufftManager();
 
-    int planIndex = planner.findIfPlanExists(key_string);
+    int planIndex = planner.find(key_string);
 
     // if found a valid plan, return it
     if (planIndex!=-1) {
-        plan = planner.getPlan(planIndex);
-        return;
+        return planner.get(planIndex);
     }
 
-    cufftHandle temp;
-    cufftResult res = cufftPlanMany(&temp, rank, n,
-                                    inembed, istride, idist,
-                                    onembed, ostride, odist,
+    PlanType retVal;
+    cufftResult res = cufftPlanMany(&retVal, rank, n,
+                                    inembed, istride, idist, onembed, ostride, odist,
                                     type, batch);
 
     // If plan creation fails, clean up the memory we hold on to and try again
     if (res != CUFFT_SUCCESS) {
         cuda::garbageCollect();
-        CUFFT_CHECK(cufftPlanMany(&temp, rank, n,
-                                  inembed, istride, idist,
-                                  onembed, ostride, odist,
+        CUFFT_CHECK(cufftPlanMany(&retVal, rank, n,
+                                  inembed, istride, idist, onembed, ostride, odist,
                                   type, batch));
     }
 
-    plan = temp;
-    cufftSetStream(plan, cuda::getStream(cuda::getActiveDeviceId()));
+    cufftSetStream(retVal, cuda::getStream(cuda::getActiveDeviceId()));
 
     // push the plan into plan cache
-    planner.pushPlan(key_string, plan);
+    planner.push(key_string, retVal);
+
+    return retVal;
 }
 
 }
diff --git a/src/backend/cuda/cufft.hpp b/src/backend/cuda/cufft.hpp
new file mode 100644
index 0000000000..6142004151
--- /dev/null
+++ b/src/backend/cuda/cufft.hpp
@@ -0,0 +1,59 @@
+/*******************************************************
+ * Copyright (c) 2016, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+#include <err_common.hpp>
+#include <cufft.h>
+#include <common/FFTPlanCache.hpp>
+#include <cstdio>
+
+namespace cuda
+{
+typedef cufftHandle PlanType; //used in platform.hpp
+
+const char * _cufftGetResultString(cufftResult res);
+
+PlanType findPlan(int rank, int *n,
+                  int *inembed, int istride, int idist,
+                  int *onembed, int ostride, int odist,
+                  cufftType type, int batch);
+
+class PlanCache : public common::FFTPlanCache<PlanCache, PlanType>
+{
+    friend PlanType findPlan(int rank, int *n,
+                             int *inembed, int istride, int idist,
+                             int *onembed, int ostride, int odist,
+                             cufftType type, int batch);
+
+    public:
+        PlanCache() {}
+        void initLibrary() {}
+        void deInitLibrary() {}
+
+        void removePlan(PlanType plan) {
+            cufftDestroy(plan);
+        }
+};
+}
+
+#define CUFFT_CHECK(fn) do {                        \
+        cufftResult _cufft_res = fn;                \
+        if (_cufft_res != CUFFT_SUCCESS) {          \
+            char cufft_res_msg[1024];               \
+            snprintf(cufft_res_msg,                 \
+                     sizeof(cufft_res_msg),         \
+                     "cuFFT Error (%d): %s\n",      \
+                     (int)(_cufft_res),             \
+                     cuda::_cufftGetResultString(   \
+                         _cufft_res));              \
+                                                    \
+            AF_ERROR(cufft_res_msg,                 \
+                     AF_ERR_INTERNAL);              \
+        }                                           \
+    } while(0)
diff --git a/src/backend/cuda/cufftManager.hpp b/src/backend/cuda/cufftManager.hpp
deleted file mode 100644
index 2d78f891c2..0000000000
--- a/src/backend/cuda/cufftManager.hpp
+++ /dev/null
@@ -1,121 +0,0 @@
-/*******************************************************
- * Copyright (c) 2016, ArrayFire
- * All rights reserved.
- *
- * This file is distributed under 3-clause BSD license.
- * The complete license agreement can be obtained at:
- * http://arrayfire.com/licenses/BSD-3-Clause
- ********************************************************/
-
-#pragma once
-#include <err_common.hpp>
-#include <cufft.h>
-
-#include <cstdio>
-#include <deque>
-#include <string>
-#include <utility>
-
-namespace cuda
-{
-class DeviceManager;
-}
-
-namespace cufft
-{
-
-typedef std::pair<std::string, cufftHandle> FFTPlanPair;
-typedef std::deque<FFTPlanPair> FFTPlanCache;
-
-const char * _cufftGetResultString(cufftResult res);
-
-void findPlan(cufftHandle &plan, int rank, int *n,
-              int *inembed, int istride, int idist,
-              int *onembed, int ostride, int odist,
-              cufftType type, int batch);
-
-// cuFFTPlanner caches fft plans
-//
-// new plan |--> IF number of plans cached is at limit, pop the least used entry and push new plan.
-//          |
-//          |--> ELSE just push the plan
-// existing plan -> reuse a plan
-class cuFFTPlanner
-{
-    friend class cuda::DeviceManager;
-
-    friend void findPlan(cufftHandle &plan, int rank, int *n,
-                         int *inembed, int istride, int idist,
-                         int *onembed, int ostride, int odist,
-                         cufftType type, int batch);
-
-    public:
-        inline void setMaxCacheSize(size_t size) {
-            mCache.resize(size, FFTPlanPair(std::string(""), 0));
-        }
-
-        inline size_t getMaxCacheSize() const {
-            return mMaxCacheSize;
-        }
-
-        inline cufftHandle getPlan(int index) const {
-            return mCache[index].second;
-        }
-
-        // iterates through plan cache from front to back
-        // of the cache(queue)
-        int findIfPlanExists(std::string keyString) const {
-            int retVal = -1;
-            for(uint i=0; i<mCache.size(); ++i) {
-                if (keyString == mCache[i].first) {
-                    retVal = i;
-                }
-            }
-            return retVal;
-        }
-
-        // pops plan from the back of cache(queue)
-        void popPlan() {
-            if (!mCache.empty()) {
-                // destroy the cufft plan associated with the
-                // least recently used plan
-                cufftDestroy(mCache.back().second);
-                // now pop the entry from cache
-                mCache.pop_back();
-            }
-        }
-
-        // pushes plan to the front of cache(queue)
-        void pushPlan(std::string keyString, cufftHandle plan) {
-            if (mCache.size()>mMaxCacheSize) {
-                popPlan();
-            }
-            mCache.push_front(FFTPlanPair(keyString, plan));
-        }
-
-    private:
-        cuFFTPlanner() : mMaxCacheSize(5) {}
-        cuFFTPlanner(cuFFTPlanner const&);
-        void operator=(cuFFTPlanner const&);
-
-        size_t       mMaxCacheSize;
-        FFTPlanCache mCache;
-};
-
-}
-
-#define CUFFT_CHECK(fn) do {                        \
-        cufftResult _cufft_res = fn;                \
-        if (_cufft_res != CUFFT_SUCCESS) {          \
-            char cufft_res_msg[1024];               \
-            snprintf(cufft_res_msg,                 \
-                     sizeof(cufft_res_msg),         \
-                     "cuFFT Error (%d): %s\n",      \
-                     (int)(_cufft_res),             \
-                     cufft::_cufftGetResultString(  \
-                         _cufft_res));              \
-                                                    \
-            AF_ERROR(cufft_res_msg,                 \
-                     AF_ERR_INTERNAL);              \
-        }                                           \
-    } while(0)
diff --git a/src/backend/cuda/fft.cpp b/src/backend/cuda/fft.cpp
index 189033e8ae..19a753421a 100644
--- a/src/backend/cuda/fft.cpp
+++ b/src/backend/cuda/fft.cpp
@@ -12,7 +12,7 @@
 #include <copy.hpp>
 #include <fft.hpp>
 #include <debug_cuda.hpp>
-#include <cufftManager.hpp>
+#include <cufft.hpp>
 #include <math.hpp>
 #include <memory.hpp>
 
@@ -21,10 +21,9 @@ using std::string;
 
 namespace cuda
 {
-
 void setFFTPlanCacheSize(size_t numPlans)
 {
-    cuda::getcufftPlanManager().setMaxCacheSize(numPlans);
+    cufftManager().maxCacheSize(numPlans);
 }
 
 template<typename T>
@@ -89,11 +88,10 @@ void fft_inplace(Array<T> &in)
         batch *= idims[i];
     }
 
-    cufftHandle plan;
-    cufft::findPlan(plan, rank, t_dims,
-                    in_embed , istrides[0], istrides[rank],
-                    in_embed , istrides[0], istrides[rank],
-                    (cufftType)cufft_transform<T>::type, batch);
+    cufftHandle plan = findPlan(rank, t_dims,
+                                in_embed , istrides[0], istrides[rank],
+                                in_embed , istrides[0], istrides[rank],
+                                (cufftType)cufft_transform<T>::type, batch);
 
     cufft_transform<T> transform;
     CUFFT_CHECK(transform(plan, (T *)in.get(), in.get(), direction ? CUFFT_FORWARD : CUFFT_INVERSE));
@@ -124,11 +122,10 @@ Array<Tc> fft_r2c(const Array<Tr> &in)
     dim4 istrides = in.strides();
     dim4 ostrides = out.strides();
 
-    cufftHandle plan;
-    cufft::findPlan(plan, rank, t_dims,
-                    in_embed  , istrides[0], istrides[rank],
-                    out_embed , ostrides[0], ostrides[rank],
-                    (cufftType)cufft_real_transform<Tc, Tr>::type, batch);
+    cufftHandle plan = findPlan(rank, t_dims,
+                                in_embed  , istrides[0], istrides[rank],
+                                out_embed , ostrides[0], ostrides[rank],
+                                (cufftType)cufft_real_transform<Tc, Tr>::type, batch);
 
     cufft_real_transform<Tc, Tr> transform;
     CUFFT_CHECK(transform(plan, (Tr *)in.get(), out.get()));
@@ -157,11 +154,10 @@ Array<Tr> fft_c2r(const Array<Tc> &in, const dim4 &odims)
 
     cufft_real_transform<Tr, Tc> transform;
 
-    cufftHandle plan;
-    cufft::findPlan(plan, rank, t_dims,
-                    in_embed  , istrides[0], istrides[rank],
-                    out_embed , ostrides[0], ostrides[rank],
-                    (cufftType)cufft_real_transform<Tr, Tc>::type, batch);
+    cufftHandle plan = findPlan(rank, t_dims,
+                                in_embed  , istrides[0], istrides[rank],
+                                out_embed , ostrides[0], ostrides[rank],
+                                (cufftType)cufft_real_transform<Tr, Tc>::type, batch);
 
     CUFFT_CHECK(transform(plan, (Tc *)in.get(), out.get()));
     return out;
diff --git a/src/backend/cuda/platform.cpp b/src/backend/cuda/platform.cpp
index 447a3d03c9..ab4949f154 100644
--- a/src/backend/cuda/platform.cpp
+++ b/src/backend/cuda/platform.cpp
@@ -419,9 +419,9 @@ GraphicsManager& interopManager()
     return *(inst.gfxManagers[id].get());
 }
 
-cufft::cuFFTPlanner& getcufftPlanManager()
+FFTManager& cufftManager()
 {
-    return DeviceManager::getInstance().cufftManagers[cuda::getActiveDeviceId()];
+    return DeviceManager::getInstance().cufftManagers[getActiveDeviceId()];
 }
 
 cublasHandle_t getcublasHandle()
diff --git a/src/backend/cuda/platform.hpp b/src/backend/cuda/platform.hpp
index 03ab23f5b1..a18e43b215 100644
--- a/src/backend/cuda/platform.hpp
+++ b/src/backend/cuda/platform.hpp
@@ -16,7 +16,7 @@
 #include <string>
 
 #include <GraphicsResourceManager.hpp>
-#include <cufftManager.hpp>
+#include <cufft.hpp>
 #include <cublasManager.hpp>
 #include <cusolverDnManager.hpp>
 #include <cusparseManager.hpp>
@@ -81,7 +81,8 @@ MemoryManagerPinned& getMemoryManagerPinned();
 typedef common::InteropManager<GraphicsResourceManager, CGR_t> GraphicsManager;
 GraphicsManager& interopManager();
 
-cufft::cuFFTPlanner& getcufftPlanManager();
+typedef common::FFTPlanCache<PlanCache, PlanType> FFTManager;
+FFTManager& cufftManager();
 
 cublasHandle_t getcublasHandle();
 
@@ -106,7 +107,7 @@ class DeviceManager
 
         friend GraphicsManager& interopManager();
 
-        friend cufft::cuFFTPlanner& getcufftPlanManager();
+        friend FFTManager& cufftManager();
 
         friend cublasHandle_t getcublasHandle();
 
@@ -173,7 +174,7 @@ class DeviceManager
 
         std::unique_ptr<GraphicsManager> gfxManagers[MAX_DEVICES];
 
-        cufft::cuFFTPlanner cufftManagers[MAX_DEVICES];
+        FFTManager cufftManagers[MAX_DEVICES];
 
         std::unique_ptr<cublas::cublasHandle> cublasHandles[MAX_DEVICES];
 
diff --git a/src/backend/opencl/clfftManager.cpp b/src/backend/opencl/clfft.cpp
similarity index 79%
rename from src/backend/opencl/clfftManager.cpp
rename to src/backend/opencl/clfft.cpp
index ef08442474..dae1558fbe 100644
--- a/src/backend/opencl/clfftManager.cpp
+++ b/src/backend/opencl/clfft.cpp
@@ -8,18 +8,15 @@
  ********************************************************/
 
 #include <af/defines.h>
-
 #include <err_common.hpp>
-#include <clfftManager.hpp>
+#include <clfft.hpp>
 #include <platform.hpp>
-
 #include <string>
 
 using std::string;
 
-namespace clfft
+namespace opencl
 {
-
 const char * _clfftGetResultString(clfftStatus st)
 {
     switch (st)
@@ -86,12 +83,11 @@ const char * _clfftGetResultString(clfftStatus st)
     return "Unknown error";
 }
 
-void findPlan(clfftPlanHandle &plan,
-              clfftLayout iLayout, clfftLayout oLayout,
-              clfftDim rank, size_t *clLengths,
-              size_t *istrides, size_t idist,
-              size_t *ostrides, size_t odist,
-              clfftPrecision precision, size_t batch)
+PlanType findPlan(clfftLayout iLayout, clfftLayout oLayout,
+                  clfftDim rank, size_t *clLengths,
+                  size_t *istrides, size_t idist,
+                  size_t *ostrides, size_t odist,
+                  clfftPrecision precision, size_t batch)
 {
     // create the key string
     char key_str_temp[64];
@@ -126,56 +122,53 @@ void findPlan(clfftPlanHandle &plan,
     sprintf(key_str_temp, "%d:" SIZE_T_FRMT_SPECIFIER, (int)precision, batch);
     key_string.append(std::string(key_str_temp));
 
-    // find the matching plan_index in the array clFFTPlanner::mKeys
-    clFFTPlanner &planner = opencl::getclfftPlanManager();
+    FFTManager &planner = opencl::clfftManager();
 
-    int planIndex = planner.findIfPlanExists(key_string);
+    int planIndex = planner.find(key_string);
 
     // if found a valid plan, return it
     if (planIndex!=-1) {
-        plan = planner.getPlan(planIndex);
-        return;
+        return planner.get(planIndex);
     }
 
-    clfftPlanHandle temp;
+    PlanType retVal;
 
     // getContext() returns object of type Context
     // Context() returns the actual cl_context handle
-    CLFFT_CHECK(clfftCreateDefaultPlan(&temp, opencl::getContext()(), rank, clLengths));
+    CLFFT_CHECK(clfftCreateDefaultPlan(&retVal, opencl::getContext()(), rank, clLengths));
 
     // complex to complex
     if (iLayout == oLayout) {
-        CLFFT_CHECK(clfftSetResultLocation(temp, CLFFT_INPLACE));
+        CLFFT_CHECK(clfftSetResultLocation(retVal, CLFFT_INPLACE));
     } else {
-        CLFFT_CHECK(clfftSetResultLocation(temp, CLFFT_OUTOFPLACE));
+        CLFFT_CHECK(clfftSetResultLocation(retVal, CLFFT_OUTOFPLACE));
     }
 
-    CLFFT_CHECK(clfftSetLayout(temp, iLayout, oLayout));
-    CLFFT_CHECK(clfftSetPlanBatchSize(temp, batch));
-    CLFFT_CHECK(clfftSetPlanDistance(temp, idist, odist));
-    CLFFT_CHECK(clfftSetPlanInStride(temp, rank, istrides));
-    CLFFT_CHECK(clfftSetPlanOutStride(temp, rank, ostrides));
-    CLFFT_CHECK(clfftSetPlanPrecision(temp, precision));
-    CLFFT_CHECK(clfftSetPlanScale(temp, CLFFT_BACKWARD, 1.0));
+    CLFFT_CHECK(clfftSetLayout(retVal, iLayout, oLayout));
+    CLFFT_CHECK(clfftSetPlanBatchSize(retVal, batch));
+    CLFFT_CHECK(clfftSetPlanDistance(retVal, idist, odist));
+    CLFFT_CHECK(clfftSetPlanInStride(retVal, rank, istrides));
+    CLFFT_CHECK(clfftSetPlanOutStride(retVal, rank, ostrides));
+    CLFFT_CHECK(clfftSetPlanPrecision(retVal, precision));
+    CLFFT_CHECK(clfftSetPlanScale(retVal, CLFFT_BACKWARD, 1.0));
 
     // getQueue() returns object of type CommandQueue
     // CommandQueue() returns the actual cl_command_queue handle
-    CLFFT_CHECK(clfftBakePlan(temp, 1, &(opencl::getQueue()()), NULL, NULL));
-
-    plan = temp;
+    CLFFT_CHECK(clfftBakePlan(retVal, 1, &(opencl::getQueue()()), NULL, NULL));
 
     // push the plan into plan cache
-    planner.pushPlan(key_string, plan);
+    planner.push(key_string, retVal);
+
+    return retVal;
 }
 
-clFFTPlanner::clFFTPlanner()
-    : mMaxCacheSize(5)
+void PlanCache::initLibrary()
 {
     CLFFT_CHECK(clfftInitSetupData(&mFFTSetup));
     CLFFT_CHECK(clfftSetup(&mFFTSetup));
 }
 
-clFFTPlanner::~clFFTPlanner()
+void PlanCache::deInitLibrary()
 {
     //TODO: FIXME:
     // clfftTeardown() causes a "Pure Virtual Function Called" crash on
@@ -190,15 +183,8 @@ clFFTPlanner::~clFFTPlanner()
     #endif
 }
 
-void clFFTPlanner::popPlan()
+void PlanCache::removePlan(PlanType plan)
 {
-    if (!mCache.empty()) {
-        // destroy the clfft plan associated with the
-        // least recently used plan
-        CLFFT_CHECK(clfftDestroyPlan(&mCache.back().second));
-        // now pop the entry from cache
-        mCache.pop_back();
-    }
+     CLFFT_CHECK(clfftDestroyPlan(&plan));
 }
-
 }
diff --git a/src/backend/opencl/clfft.hpp b/src/backend/opencl/clfft.hpp
new file mode 100644
index 0000000000..92267e27c7
--- /dev/null
+++ b/src/backend/opencl/clfft.hpp
@@ -0,0 +1,67 @@
+/*******************************************************
+ * Copyright (c) 2016, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <clFFT.h>
+#include <common/FFTPlanCache.hpp>
+#include <memory.hpp>
+#include <cstdio>
+
+namespace opencl
+{
+typedef clfftPlanHandle PlanType;
+
+const char * _clfftGetResultString(clfftStatus st);
+
+PlanType findPlan(clfftLayout iLayout, clfftLayout oLayout,
+                  clfftDim rank, size_t *clLengths,
+                  size_t *istrides, size_t idist,
+                  size_t *ostrides, size_t odist,
+                  clfftPrecision precision, size_t batch);
+
+class PlanCache : public common::FFTPlanCache<PlanCache, PlanType>
+{
+    friend PlanType findPlan(clfftLayout iLayout, clfftLayout oLayout,
+                             clfftDim rank, size_t *clLengths,
+                             size_t *istrides, size_t idist,
+                             size_t *ostrides, size_t odist,
+                             clfftPrecision precision, size_t batch);
+
+    public:
+        PlanCache() {}
+        void initLibrary();
+        void deInitLibrary();
+
+        void removePlan(PlanType plan);
+
+    private:
+        clfftSetupData  mFFTSetup;
+};
+}
+
+#define CLFFT_CHECK(fn) do {                        \
+        clfftStatus _clfft_st = fn;                 \
+        if (_clfft_st != CLFFT_SUCCESS) {           \
+            opencl::garbageCollect();               \
+            _clfft_st = (fn);                       \
+        }                                           \
+        if (_clfft_st != CLFFT_SUCCESS) {           \
+            char clfft_st_msg[1024];                \
+            snprintf(clfft_st_msg,                  \
+                     sizeof(clfft_st_msg),          \
+                     "clFFT Error (%d): %s\n",      \
+                     (int)(_clfft_st),              \
+                     opencl::_clfftGetResultString( \
+                         _clfft_st));               \
+                                                    \
+            AF_ERROR(clfft_st_msg,                  \
+                     AF_ERR_INTERNAL);              \
+        }                                           \
+    } while(0)
diff --git a/src/backend/opencl/clfftManager.hpp b/src/backend/opencl/clfftManager.hpp
deleted file mode 100644
index 37ffd0fda0..0000000000
--- a/src/backend/opencl/clfftManager.hpp
+++ /dev/null
@@ -1,127 +0,0 @@
-/*******************************************************
- * Copyright (c) 2016, ArrayFire
- * All rights reserved.
- *
- * This file is distributed under 3-clause BSD license.
- * The complete license agreement can be obtained at:
- * http://arrayfire.com/licenses/BSD-3-Clause
- ********************************************************/
-
-#pragma once
-
-#include <clFFT.h>
-
-#include <memory.hpp>
-
-#include <cstdio>
-#include <deque>
-#include <string>
-#include <utility>
-
-namespace opencl
-{
-class DeviceManager;
-}
-
-namespace clfft
-{
-
-typedef std::pair<std::string, clfftPlanHandle> FFTPlanPair;
-typedef std::deque<FFTPlanPair> FFTPlanCache;
-
-const char * _clfftGetResultString(clfftStatus st);
-
-void findPlan(clfftPlanHandle &plan,
-              clfftLayout iLayout, clfftLayout oLayout,
-              clfftDim rank, size_t *clLengths,
-              size_t *istrides, size_t idist,
-              size_t *ostrides, size_t odist,
-              clfftPrecision precision, size_t batch);
-
-// clFFTPlanner caches fft plans
-//
-// new plan |--> IF number of plans cached is at limit, pop the least used entry and push new plan.
-//          |
-//          |--> ELSE just push the plan
-// existing plan -> reuse a plan
-class clFFTPlanner
-{
-    friend class opencl::DeviceManager;
-
-    friend void findPlan(clfftPlanHandle &plan,
-                         clfftLayout iLayout, clfftLayout oLayout,
-                         clfftDim rank, size_t *clLengths,
-                         size_t *istrides, size_t idist,
-                         size_t *ostrides, size_t odist,
-                         clfftPrecision precision, size_t batch);
-
-    public:
-        clFFTPlanner();
-        ~clFFTPlanner();
-
-        inline void setMaxCacheSize(size_t size) {
-            mCache.resize(size, FFTPlanPair(std::string(""), 0));
-        }
-
-        inline size_t getMaxCacheSize() const {
-            return mMaxCacheSize;
-        }
-
-        inline clfftPlanHandle getPlan(int index) const {
-            return mCache[index].second;
-        }
-
-        // iterates through plan cache from front to back
-        // of the cache(queue)
-        int findIfPlanExists(std::string keyString) const {
-            int retVal = -1;
-            for(uint i=0; i<mCache.size(); ++i) {
-                if (keyString == mCache[i].first) {
-                    retVal = i;
-                }
-            }
-            return retVal;
-        }
-
-        // pops plan from the back of cache(queue)
-        void popPlan();
-
-        // pushes plan to the front of cache(queue)
-        void pushPlan(std::string keyString, clfftPlanHandle plan) {
-            if (mCache.size()>mMaxCacheSize) {
-                popPlan();
-            }
-            mCache.push_front(FFTPlanPair(keyString, plan));
-        }
-
-    private:
-        clFFTPlanner(clFFTPlanner const&);
-        void operator=(clFFTPlanner const&);
-
-        clfftSetupData  mFFTSetup;
-
-        size_t       mMaxCacheSize;
-        FFTPlanCache mCache;
-};
-
-}
-
-#define CLFFT_CHECK(fn) do {                        \
-        clfftStatus _clfft_st = fn;                 \
-        if (_clfft_st != CLFFT_SUCCESS) {           \
-            opencl::garbageCollect();               \
-            _clfft_st = (fn);                       \
-        }                                           \
-        if (_clfft_st != CLFFT_SUCCESS) {           \
-            char clfft_st_msg[1024];                \
-            snprintf(clfft_st_msg,                  \
-                     sizeof(clfft_st_msg),          \
-                     "clFFT Error (%d): %s\n",      \
-                     (int)(_clfft_st),              \
-                     clfft::_clfftGetResultString(  \
-                         _clfft_st));               \
-                                                    \
-            AF_ERROR(clfft_st_msg,                  \
-                     AF_ERR_INTERNAL);              \
-        }                                           \
-    } while(0)
diff --git a/src/backend/opencl/fft.cpp b/src/backend/opencl/fft.cpp
index c5a550304d..f3423ec6c0 100644
--- a/src/backend/opencl/fft.cpp
+++ b/src/backend/opencl/fft.cpp
@@ -14,7 +14,7 @@
 #include <err_opencl.hpp>
 #include <math.hpp>
 #include <memory.hpp>
-#include <clfftManager.hpp>
+#include <clfft.hpp>
 
 using af::dim4;
 using std::string;
@@ -24,7 +24,7 @@ namespace opencl
 
 void setFFTPlanCacheSize(size_t numPlans)
 {
-    getclfftPlanManager().setMaxCacheSize(numPlans);
+    clfftManager().maxCacheSize(numPlans);
 }
 
 template<typename T> struct Precision;
@@ -78,21 +78,15 @@ void fft_inplace(Array<T> &in)
     computeDims(tdims   , in.dims());
     computeDims(istrides, in.strides());
 
-    clfftPlanHandle plan;
-
     int batch = 1;
     for (int i = rank; i < 4; i++) {
         batch *= tdims[i];
     }
 
-    clfft::findPlan(plan,
-                    CLFFT_COMPLEX_INTERLEAVED,
-                    CLFFT_COMPLEX_INTERLEAVED,
-                    (clfftDim)rank, tdims,
-                    istrides, istrides[rank],
-                    istrides, istrides[rank],
-                    (clfftPrecision)Precision<T>::type,
-                    batch);
+    PlanType plan = findPlan(CLFFT_COMPLEX_INTERLEAVED, CLFFT_COMPLEX_INTERLEAVED,
+                             (clfftDim)rank, tdims,
+                             istrides, istrides[rank], istrides, istrides[rank],
+                             (clfftPrecision)Precision<T>::type, batch);
 
     cl_mem imem = (*in.get())();
     cl_command_queue queue = getQueue()();
@@ -119,21 +113,15 @@ Array<Tc> fft_r2c(const Array<Tr> &in)
     computeDims(istrides,  in.strides());
     computeDims(ostrides, out.strides());
 
-    clfftPlanHandle plan;
-
     int batch = 1;
     for (int i = rank; i < 4; i++) {
         batch *= tdims[i];
     }
 
-    clfft::findPlan(plan,
-                    CLFFT_REAL,
-                    CLFFT_HERMITIAN_INTERLEAVED,
-                    (clfftDim)rank, tdims,
-                    istrides, istrides[rank],
-                    ostrides, ostrides[rank],
-                    (clfftPrecision)Precision<Tc>::type,
-                    batch);
+    PlanType plan = findPlan(CLFFT_REAL, CLFFT_HERMITIAN_INTERLEAVED,
+                             (clfftDim)rank, tdims,
+                             istrides, istrides[rank], ostrides, ostrides[rank],
+                             (clfftPrecision)Precision<Tc>::type, batch);
 
     cl_mem imem = (*in.get())();
     cl_mem omem = (*out.get())();
@@ -159,21 +147,15 @@ Array<Tr> fft_c2r(const Array<Tc> &in, const dim4 &odims)
     computeDims(istrides,  in.strides());
     computeDims(ostrides, out.strides());
 
-    clfftPlanHandle plan;
-
     int batch = 1;
     for (int i = rank; i < 4; i++) {
         batch *= tdims[i];
     }
 
-    clfft::findPlan(plan,
-                    CLFFT_HERMITIAN_INTERLEAVED,
-                    CLFFT_REAL,
-                    (clfftDim)rank, tdims,
-                    istrides, istrides[rank],
-                    ostrides, ostrides[rank],
-                    (clfftPrecision)Precision<Tc>::type,
-                    batch);
+    PlanType plan = findPlan(CLFFT_HERMITIAN_INTERLEAVED, CLFFT_REAL,
+                             (clfftDim)rank, tdims,
+                             istrides, istrides[rank], ostrides, ostrides[rank],
+                             (clfftPrecision)Precision<Tc>::type, batch);
 
     cl_mem imem = (*in.get())();
     cl_mem omem = (*out.get())();
diff --git a/src/backend/opencl/platform.cpp b/src/backend/opencl/platform.cpp
index 4c263e57ad..31b3abafa3 100644
--- a/src/backend/opencl/platform.cpp
+++ b/src/backend/opencl/platform.cpp
@@ -877,7 +877,7 @@ GraphicsManager& interopManager()
     return *(inst.gfxManagers[id].get());
 }
 
-clfft::clFFTPlanner& getclfftPlanManager()
+FFTManager& clfftManager()
 {
     return DeviceManager::getInstance().clfftManagers[getActiveDeviceId()];
 }
diff --git a/src/backend/opencl/platform.hpp b/src/backend/opencl/platform.hpp
index 9badbe8a58..d94d60ae43 100644
--- a/src/backend/opencl/platform.hpp
+++ b/src/backend/opencl/platform.hpp
@@ -27,7 +27,7 @@
 #include <string>
 
 #include <GraphicsResourceManager.hpp>
-#include <clfftManager.hpp>
+#include <clfft.hpp>
 
 namespace opencl
 {
@@ -95,7 +95,8 @@ MemoryManagerPinned& getMemoryManagerPinned();
 typedef common::InteropManager<GraphicsResourceManager, CGR_t> GraphicsManager;
 GraphicsManager& interopManager();
 
-clfft::clFFTPlanner& getclfftPlanManager();
+typedef common::FFTPlanCache<PlanCache, PlanType> FFTManager;
+FFTManager& clfftManager();
 //
 ///////////////////////// END Sub-Managers /////////////////////
 
@@ -107,7 +108,7 @@ class DeviceManager
 
     friend GraphicsManager& interopManager();
 
-    friend clfft::clFFTPlanner& getclfftPlanManager();
+    friend FFTManager& clfftManager();
 
     friend std::string getDeviceInfo();
 
@@ -181,6 +182,6 @@ class DeviceManager
         std::unique_ptr<MemoryManagerPinned> pinnedMemManager;
 
         std::unique_ptr<GraphicsManager> gfxManagers[MAX_DEVICES];
-        clfft::clFFTPlanner clfftManagers[MAX_DEVICES];
+        FFTManager clfftManagers[MAX_DEVICES];
 };
 }

From be345da187afbb0cbba05ae117400db3702935e0 Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Sat, 14 Jan 2017 17:08:15 +0530
Subject: [PATCH 21/82] Abstract cu{sparse|blas|solve} handle RAII wrapper

---
 src/backend/common/MatrixAlgebraHandle.hpp    | 39 +++++++++++++
 src/backend/cuda/blas.cpp                     |  8 +--
 src/backend/cuda/cholesky.cu                  |  4 +-
 src/backend/cuda/cublas.cpp                   | 40 +++++++++++++
 .../cuda/{cublasManager.hpp => cublas.hpp}    | 48 ++++++----------
 src/backend/cuda/cublasManager.cpp            | 53 ------------------
 src/backend/cuda/cusolverDn.cpp               | 36 ++++++++++++
 .../{cusolverDnManager.hpp => cusolverDn.hpp} | 48 ++++++----------
 src/backend/cuda/cusolverDnManager.cpp        | 56 -------------------
 src/backend/cuda/cusparse.cpp                 | 39 +++++++++++++
 .../{cusparseManager.hpp => cusparse.hpp}     | 48 ++++++----------
 src/backend/cuda/cusparseManager.cpp          | 54 ------------------
 src/backend/cuda/lu.cu                        |  4 +-
 src/backend/cuda/platform.cpp                 | 33 +++--------
 src/backend/cuda/platform.hpp                 | 30 ++++------
 src/backend/cuda/qr.cu                        | 10 ++--
 src/backend/cuda/solve.cu                     | 16 +++---
 src/backend/cuda/sparse.cu                    | 30 +++++-----
 src/backend/cuda/sparse_blas.cpp              |  4 +-
 src/backend/cuda/svd.cu                       |  6 +-
 20 files changed, 267 insertions(+), 339 deletions(-)
 create mode 100644 src/backend/common/MatrixAlgebraHandle.hpp
 create mode 100644 src/backend/cuda/cublas.cpp
 rename src/backend/cuda/{cublasManager.hpp => cublas.hpp} (68%)
 delete mode 100644 src/backend/cuda/cublasManager.cpp
 create mode 100644 src/backend/cuda/cusolverDn.cpp
 rename src/backend/cuda/{cusolverDnManager.hpp => cusolverDn.hpp} (68%)
 delete mode 100644 src/backend/cuda/cusolverDnManager.cpp
 create mode 100644 src/backend/cuda/cusparse.cpp
 rename src/backend/cuda/{cusparseManager.hpp => cusparse.hpp} (70%)
 delete mode 100644 src/backend/cuda/cusparseManager.cpp

diff --git a/src/backend/common/MatrixAlgebraHandle.hpp b/src/backend/common/MatrixAlgebraHandle.hpp
new file mode 100644
index 0000000000..bf4f6c1110
--- /dev/null
+++ b/src/backend/common/MatrixAlgebraHandle.hpp
@@ -0,0 +1,39 @@
+/*******************************************************
+ * Copyright (c) 2016, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <err_common.hpp>
+#include <cstdio>
+
+namespace common
+{
+template<typename T, typename H>
+class MatrixAlgebraHandle
+{
+    public:
+        MatrixAlgebraHandle() {
+            static_cast<T*>(this)->createHandle(&handle);
+        }
+
+        ~MatrixAlgebraHandle() {
+            static_cast<T*>(this)->destroyHandle(handle);
+        }
+
+        H get() const {
+            return handle;
+        }
+
+    private:
+        MatrixAlgebraHandle(MatrixAlgebraHandle const&);
+        void operator=(MatrixAlgebraHandle const&);
+
+        H handle;
+};
+}
diff --git a/src/backend/cuda/blas.cpp b/src/backend/cuda/blas.cpp
index 3d9d3aba78..46e4bfa8fb 100644
--- a/src/backend/cuda/blas.cpp
+++ b/src/backend/cuda/blas.cpp
@@ -17,7 +17,7 @@
 #include <cassert>
 #include <math.hpp>
 #include <err_common.hpp>
-#include <cublasManager.hpp>
+#include <cublas.hpp>
 #include <arith.hpp>
 #include <reduce.hpp>
 #include <complex.hpp>
@@ -171,7 +171,7 @@ Array<T> matmul(const Array<T> &lhs, const Array<T> &rhs,
     if(rDims[bColDim] == 1) {
         N = lDims[aColDim];
         CUBLAS_CHECK(gemv_func<T>()(
-                         getcublasHandle(),
+                         cublasHandle(),
                          lOpts,
                          lDims[0],
                          lDims[1],
@@ -182,7 +182,7 @@ Array<T> matmul(const Array<T> &lhs, const Array<T> &rhs,
                          out.get(), 1));
     } else {
         CUBLAS_CHECK(gemm_func<T>()(
-                         getcublasHandle(),
+                         cublasHandle(),
                          lOpts,
                          rOpts,
                          M, N, K,
@@ -224,7 +224,7 @@ void trsm(const Array<T> &lhs, Array<T> &rhs, af_mat_prop trans,
     dim4 rStrides = rhs.strides();
 
     CUBLAS_CHECK(trsm_func<T>()(
-                     getcublasHandle(),
+                     cublasHandle(),
                      is_left  ? CUBLAS_SIDE_LEFT : CUBLAS_SIDE_RIGHT,
                      is_upper ? CUBLAS_FILL_MODE_UPPER : CUBLAS_FILL_MODE_LOWER,
                      toCblasTranspose(trans),
diff --git a/src/backend/cuda/cholesky.cu b/src/backend/cuda/cholesky.cu
index 1c9b3c23ea..53df19b67f 100644
--- a/src/backend/cuda/cholesky.cu
+++ b/src/backend/cuda/cholesky.cu
@@ -110,7 +110,7 @@ int cholesky_inplace(Array<T> &in, const bool is_upper)
     if(is_upper)
         uplo = CUBLAS_FILL_MODE_UPPER;
 
-    CUSOLVER_CHECK(potrf_buf_func<T>()(getcusolverDnHandle(),
+    CUSOLVER_CHECK(potrf_buf_func<T>()(cusolverDnHandle(),
                                        uplo,
                                        N,
                                        in.get(), in.strides()[1],
@@ -119,7 +119,7 @@ int cholesky_inplace(Array<T> &in, const bool is_upper)
     T *workspace = memAlloc<T>(lwork);
     int *d_info = memAlloc<int>(1);
 
-    CUSOLVER_CHECK(potrf_func<T>()(getcusolverDnHandle(),
+    CUSOLVER_CHECK(potrf_func<T>()(cusolverDnHandle(),
                                    uplo,
                                    N,
                                    in.get(), in.strides()[1],
diff --git a/src/backend/cuda/cublas.cpp b/src/backend/cuda/cublas.cpp
new file mode 100644
index 0000000000..5e40781c0a
--- /dev/null
+++ b/src/backend/cuda/cublas.cpp
@@ -0,0 +1,40 @@
+/*******************************************************
+ * Copyright (c) 2014, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <platform.hpp>
+#include <err_common.hpp>
+#include <cublas.hpp>
+
+namespace cuda
+{
+const char *errorString(cublasStatus_t err)
+{
+    switch(err)
+    {
+    case CUBLAS_STATUS_SUCCESS         : return "CUBLAS_STATUS_SUCCESS"         ;
+    case CUBLAS_STATUS_NOT_INITIALIZED : return "CUBLAS_STATUS_NOT_INITIALIZED" ;
+    case CUBLAS_STATUS_ALLOC_FAILED    : return "CUBLAS_STATUS_ALLOC_FAILED"    ;
+    case CUBLAS_STATUS_INVALID_VALUE   : return "CUBLAS_STATUS_INVALID_VALUE"   ;
+    case CUBLAS_STATUS_ARCH_MISMATCH   : return "CUBLAS_STATUS_ARCH_MISMATCH"   ;
+    case CUBLAS_STATUS_MAPPING_ERROR   : return "CUBLAS_STATUS_MAPPING_ERROR"   ;
+    case CUBLAS_STATUS_EXECUTION_FAILED: return "CUBLAS_STATUS_EXECUTION_FAILED";
+    case CUBLAS_STATUS_INTERNAL_ERROR  : return "CUBLAS_STATUS_INTERNAL_ERROR"  ;
+#if CUDA_VERSION > 5050
+    case CUBLAS_STATUS_NOT_SUPPORTED   : return "CUBLAS_STATUS_NOT_SUPPORTED"   ;
+#endif
+    default: return "UNKNOWN";
+    }
+}
+
+void cublasHandle::createHandle(BlasHandle* handle)
+{
+    CUBLAS_CHECK(cublasCreate(handle));
+    CUBLAS_CHECK(cublasSetStream(*handle, cuda::getStream(cuda::getActiveDeviceId())));
+}
+}
diff --git a/src/backend/cuda/cublasManager.hpp b/src/backend/cuda/cublas.hpp
similarity index 68%
rename from src/backend/cuda/cublasManager.hpp
rename to src/backend/cuda/cublas.hpp
index 41fb567e04..5e5dd40922 100644
--- a/src/backend/cuda/cublasManager.hpp
+++ b/src/backend/cuda/cublas.hpp
@@ -8,43 +8,16 @@
  ********************************************************/
 
 #pragma once
-#include <stdio.h>
-#include <err_common.hpp>
-#include <defines.hpp>
 #include <cublas_v2.h>
+#include <defines.hpp>
+#include <common/MatrixAlgebraHandle.hpp>
 
 namespace cuda
 {
-
-class DeviceManager;
-
-}
-
-namespace cublas
-{
+typedef cublasHandle_t BlasHandle;
 
 const char * errorString(cublasStatus_t err);
 
-//RAII class around the cublas Handle
-class cublasHandle
-{
-    friend class cuda::DeviceManager;
-
-    public:
-        ~cublasHandle();
-        cublasHandle_t get() const;
-
-    private:
-        cublasHandle();
-        cublasHandle(cublasHandle const&);
-        void operator=(cublasHandle const&);
-
-        cublasHandle_t handle;
-};
-
-}
-
-
 #define CUBLAS_CHECK(fn) do {                   \
         cublasStatus_t _error = fn;             \
         if (_error != CUBLAS_STATUS_SUCCESS) {  \
@@ -53,10 +26,21 @@ class cublasHandle
                      sizeof(_err_msg),          \
                      "CUBLAS Error (%d): %s\n", \
                      (int)(_error),             \
-                     cublas::errorString(       \
-                         _error));              \
+                     errorString(_error));      \
                                                 \
             AF_ERROR(_err_msg,                  \
                      AF_ERR_INTERNAL);          \
         }                                       \
     } while(0)
+
+class cublasHandle : public common::MatrixAlgebraHandle<cublasHandle, BlasHandle>
+{
+    public:
+        void createHandle(BlasHandle* handle);
+        void destroyHandle(BlasHandle handle) {
+            cublasDestroy(handle);
+        }
+};
+
+typedef common::MatrixAlgebraHandle<cublasHandle, BlasHandle> BlasHandleWrapper;
+}
diff --git a/src/backend/cuda/cublasManager.cpp b/src/backend/cuda/cublasManager.cpp
deleted file mode 100644
index e00beffad1..0000000000
--- a/src/backend/cuda/cublasManager.cpp
+++ /dev/null
@@ -1,53 +0,0 @@
-/*******************************************************
- * Copyright (c) 2014, ArrayFire
- * All rights reserved.
- *
- * This file is distributed under 3-clause BSD license.
- * The complete license agreement can be obtained at:
- * http://arrayfire.com/licenses/BSD-3-Clause
- ********************************************************/
-
-#include <platform.hpp>
-#include <err_common.hpp>
-#include <cublasManager.hpp>
-
-namespace cublas
-{
-
-const char *errorString(cublasStatus_t err)
-{
-
-    switch(err)
-    {
-    case    CUBLAS_STATUS_SUCCESS:              return "CUBLAS_STATUS_SUCCESS";
-    case    CUBLAS_STATUS_NOT_INITIALIZED:      return "CUBLAS_STATUS_NOT_INITIALIZED";
-    case    CUBLAS_STATUS_ALLOC_FAILED:         return "CUBLAS_STATUS_ALLOC_FAILED";
-    case    CUBLAS_STATUS_INVALID_VALUE:        return "CUBLAS_STATUS_INVALID_VALUE";
-    case    CUBLAS_STATUS_ARCH_MISMATCH:        return "CUBLAS_STATUS_ARCH_MISMATCH";
-    case    CUBLAS_STATUS_MAPPING_ERROR:        return "CUBLAS_STATUS_MAPPING_ERROR";
-    case    CUBLAS_STATUS_EXECUTION_FAILED:     return "CUBLAS_STATUS_EXECUTION_FAILED";
-    case    CUBLAS_STATUS_INTERNAL_ERROR:       return "CUBLAS_STATUS_INTERNAL_ERROR";
-#if CUDA_VERSION > 5050
-    case    CUBLAS_STATUS_NOT_SUPPORTED:        return "CUBLAS_STATUS_NOT_SUPPORTED";
-#endif
-    default:                                    return "UNKNOWN";
-    }
-}
-
-cublasHandle::cublasHandle()  : handle(0)
-{
-    CUBLAS_CHECK(cublasCreate(&handle));
-    CUBLAS_CHECK(cublasSetStream(handle, cuda::getStream(cuda::getActiveDeviceId())));
-}
-
-cublasHandle::~cublasHandle()
-{
-    cublasDestroy(handle);
-}
-
-cublasHandle_t cublasHandle::get() const
-{
-    return handle;
-}
-
-}
diff --git a/src/backend/cuda/cusolverDn.cpp b/src/backend/cuda/cusolverDn.cpp
new file mode 100644
index 0000000000..e02f50cae2
--- /dev/null
+++ b/src/backend/cuda/cusolverDn.cpp
@@ -0,0 +1,36 @@
+/*******************************************************
+ * Copyright (c) 2014, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <cusolverDn.hpp>
+#include <platform.hpp>
+#include <debug_cuda.hpp>
+#include <stdexcept>
+#include <string>
+
+namespace cuda
+{
+const char *errorString(cusolverStatus_t err)
+{
+    switch(err) {
+    case CUSOLVER_STATUS_SUCCESS                  : return "CUSOLVER_STATUS_SUCCESS"                  ;
+    case CUSOLVER_STATUS_NOT_INITIALIZED          : return "CUSOLVER_STATUS_NOT_INITIALIZED"          ;
+    case CUSOLVER_STATUS_ALLOC_FAILED             : return "CUSOLVER_STATUS_ALLOC_FAILED"             ;
+    case CUSOLVER_STATUS_INVALID_VALUE            : return "CUSOLVER_STATUS_INVALID_VALUE"            ;
+    case CUSOLVER_STATUS_ARCH_MISMATCH            : return "CUSOLVER_STATUS_ARCH_MISMATCH"            ;
+    case CUSOLVER_STATUS_MAPPING_ERROR            : return "CUSOLVER_STATUS_MAPPING_ERROR"            ;
+    case CUSOLVER_STATUS_EXECUTION_FAILED         : return "CUSOLVER_STATUS_EXECUTION_FAILED"         ;
+    case CUSOLVER_STATUS_INTERNAL_ERROR           : return "CUSOLVER_STATUS_INTERNAL_ERROR"           ;
+    case CUSOLVER_STATUS_MATRIX_TYPE_NOT_SUPPORTED: return "CUSOLVER_STATUS_MATRIX_TYPE_NOT_SUPPORTED";
+    case CUSOLVER_STATUS_NOT_SUPPORTED            : return "CUSOLVER_STATUS_NOT_SUPPORTED"            ;
+    case CUSOLVER_STATUS_ZERO_PIVOT               : return "CUSOLVER_STATUS_ZERO_PIVOT"               ;
+    case CUSOLVER_STATUS_INVALID_LICENSE          : return "CUSOLVER_STATUS_INVALID_LICENSE"          ;
+    default: return "UNKNOWN";
+    }
+}
+}
diff --git a/src/backend/cuda/cusolverDnManager.hpp b/src/backend/cuda/cusolverDn.hpp
similarity index 68%
rename from src/backend/cuda/cusolverDnManager.hpp
rename to src/backend/cuda/cusolverDn.hpp
index c12cea178e..67592e67f0 100644
--- a/src/backend/cuda/cusolverDnManager.hpp
+++ b/src/backend/cuda/cusolverDn.hpp
@@ -11,41 +11,15 @@
 
 #include <cuda_runtime.h>
 #include <cusolverDn.h>
-#include <stdio.h>
 #include <defines.hpp>
-#include <err_common.hpp>
+#include <common/MatrixAlgebraHandle.hpp>
 
 namespace cuda
 {
-
-class DeviceManager;
-
-}
-
-namespace cusolver
-{
+typedef cusolverDnHandle_t SolveHandle;
 
 const char * errorString(cusolverStatus_t err);
 
-//RAII class around the cusolver Handle
-class cusolverDnHandle
-{
-    friend class cuda::DeviceManager;
-
-    public:
-        ~cusolverDnHandle();
-        cusolverDnHandle_t get() const;
-
-    private:
-        cusolverDnHandle();
-        cusolverDnHandle(cusolverDnHandle const&);
-        void operator=(cusolverDnHandle const&);
-
-        cusolverDnHandle_t handle;
-};
-
-}
-
 #define CUSOLVER_CHECK(fn) do {                     \
         cusolverStatus_t _error = fn;               \
         if (_error != CUSOLVER_STATUS_SUCCESS) {    \
@@ -54,10 +28,24 @@ class cusolverDnHandle
                      sizeof(_err_msg),              \
                      "CUBLAS Error (%d): %s\n",     \
                      (int)(_error),                 \
-                     cusolver::errorString(         \
-                         _error));                  \
+                     errorString(_error));          \
                                                     \
             AF_ERROR(_err_msg,                      \
                      AF_ERR_INTERNAL);              \
         }                                           \
     } while(0)
+
+class cusolverDnHandle : public common::MatrixAlgebraHandle<cusolverDnHandle, SolveHandle>
+{
+    public:
+        void createHandle(SolveHandle* handle) {
+            CUSOLVER_CHECK(cusolverDnCreate(handle));
+        }
+
+        void destroyHandle(SolveHandle handle) {
+            cusolverDnDestroy(handle);
+        }
+};
+
+typedef common::MatrixAlgebraHandle<cusolverDnHandle, SolveHandle> SolveHandleWrapper;
+}
diff --git a/src/backend/cuda/cusolverDnManager.cpp b/src/backend/cuda/cusolverDnManager.cpp
deleted file mode 100644
index 5940e929df..0000000000
--- a/src/backend/cuda/cusolverDnManager.cpp
+++ /dev/null
@@ -1,56 +0,0 @@
-/*******************************************************
- * Copyright (c) 2014, ArrayFire
- * All rights reserved.
- *
- * This file is distributed under 3-clause BSD license.
- * The complete license agreement can be obtained at:
- * http://arrayfire.com/licenses/BSD-3-Clause
- ********************************************************/
-
-#include <cusolverDnManager.hpp>
-#include <platform.hpp>
-#include <debug_cuda.hpp>
-
-#include <stdexcept>
-#include <string>
-
-namespace cusolver
-{
-
-const char *errorString(cusolverStatus_t err)
-{
-    switch(err) {
-    case    CUSOLVER_STATUS_SUCCESS                     :   return "CUSOLVER_STATUS_SUCCESS"                    ;
-    case    CUSOLVER_STATUS_NOT_INITIALIZED             :   return "CUSOLVER_STATUS_NOT_INITIALIZED"            ;
-    case    CUSOLVER_STATUS_ALLOC_FAILED                :   return "CUSOLVER_STATUS_ALLOC_FAILED"               ;
-    case    CUSOLVER_STATUS_INVALID_VALUE               :   return "CUSOLVER_STATUS_INVALID_VALUE"              ;
-    case    CUSOLVER_STATUS_ARCH_MISMATCH               :   return "CUSOLVER_STATUS_ARCH_MISMATCH"              ;
-    case    CUSOLVER_STATUS_MAPPING_ERROR               :   return "CUSOLVER_STATUS_MAPPING_ERROR"              ;
-    case    CUSOLVER_STATUS_EXECUTION_FAILED            :   return "CUSOLVER_STATUS_EXECUTION_FAILED"           ;
-    case    CUSOLVER_STATUS_INTERNAL_ERROR              :   return "CUSOLVER_STATUS_INTERNAL_ERROR"             ;
-    case    CUSOLVER_STATUS_MATRIX_TYPE_NOT_SUPPORTED   :   return "CUSOLVER_STATUS_MATRIX_TYPE_NOT_SUPPORTED"  ;
-    case    CUSOLVER_STATUS_NOT_SUPPORTED               :   return "CUSOLVER_STATUS_NOT_SUPPORTED"              ;
-    case    CUSOLVER_STATUS_ZERO_PIVOT                  :   return "CUSOLVER_STATUS_ZERO_PIVOT"                 ;
-    case    CUSOLVER_STATUS_INVALID_LICENSE             :   return "CUSOLVER_STATUS_INVALID_LICENSE"            ;
-    default                                             :   return "UNKNOWN";
-    }
-}
-
-
-cusolverDnHandle::cusolverDnHandle()
-    : handle(0)
-{
-    CUSOLVER_CHECK(cusolverDnCreate(&handle));
-}
-
-cusolverDnHandle::~cusolverDnHandle()
-{
-    cusolverDnDestroy(handle);
-}
-
-cusolverDnHandle_t cusolverDnHandle::get() const
-{
-    return handle;
-}
-
-}
diff --git a/src/backend/cuda/cusparse.cpp b/src/backend/cuda/cusparse.cpp
new file mode 100644
index 0000000000..08204f51b7
--- /dev/null
+++ b/src/backend/cuda/cusparse.cpp
@@ -0,0 +1,39 @@
+/*******************************************************
+ * Copyright (c) 2014, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <platform.hpp>
+#include <cusparse.hpp>
+#include <stdexcept>
+#include <string>
+
+namespace cuda
+{
+const char *errorString(cusparseStatus_t err)
+{
+    switch(err) {
+    case CUSPARSE_STATUS_SUCCESS                  : return "CUSPARSE_STATUS_SUCCESS"                  ;
+    case CUSPARSE_STATUS_NOT_INITIALIZED          : return "CUSPARSE_STATUS_NOT_INITIALIZED"          ;
+    case CUSPARSE_STATUS_ALLOC_FAILED             : return "CUSPARSE_STATUS_ALLOC_FAILED"             ;
+    case CUSPARSE_STATUS_INVALID_VALUE            : return "CUSPARSE_STATUS_INVALID_VALUE"            ;
+    case CUSPARSE_STATUS_ARCH_MISMATCH            : return "CUSPARSE_STATUS_ARCH_MISMATCH"            ;
+    case CUSPARSE_STATUS_MAPPING_ERROR            : return "CUSPARSE_STATUS_MAPPING_ERROR"            ;
+    case CUSPARSE_STATUS_EXECUTION_FAILED         : return "CUSPARSE_STATUS_EXECUTION_FAILED"         ;
+    case CUSPARSE_STATUS_INTERNAL_ERROR           : return "CUSPARSE_STATUS_INTERNAL_ERROR"           ;
+    case CUSPARSE_STATUS_MATRIX_TYPE_NOT_SUPPORTED: return "CUSPARSE_STATUS_MATRIX_TYPE_NOT_SUPPORTED";
+    case CUSPARSE_STATUS_ZERO_PIVOT               : return "CUSPARSE_STATUS_ZERO_PIVOT"               ;
+    default: return "UNKNOWN";
+    }
+}
+
+void cusparseHandle::createHandle(SparseHandle* handle)
+{
+    CUSPARSE_CHECK(cusparseCreate(handle));
+    CUSPARSE_CHECK(cusparseSetStream(*handle, cuda::getStream(cuda::getActiveDeviceId())));
+}
+}
diff --git a/src/backend/cuda/cusparseManager.hpp b/src/backend/cuda/cusparse.hpp
similarity index 70%
rename from src/backend/cuda/cusparseManager.hpp
rename to src/backend/cuda/cusparse.hpp
index b5b3625f18..3bbdec81d1 100644
--- a/src/backend/cuda/cusparseManager.hpp
+++ b/src/backend/cuda/cusparse.hpp
@@ -8,43 +8,16 @@
  ********************************************************/
 
 #pragma once
-#include <stdio.h>
-#include <err_common.hpp>
-#include <defines.hpp>
 #include <cusparse_v2.h>
+#include <defines.hpp>
+#include <common/MatrixAlgebraHandle.hpp>
 
 namespace cuda
 {
-
-class DeviceManager;
-
-}
-
-namespace cusparse
-{
+typedef cusparseHandle_t SparseHandle;
 
 const char * errorString(cusparseStatus_t err);
 
-//RAII class around the cusparse Handle
-class cusparseHandle
-{
-    friend class cuda::DeviceManager;
-
-    public:
-    ~cusparseHandle();
-    cusparseHandle_t get() const;
-
-    private:
-    cusparseHandle();
-    cusparseHandle(cusparseHandle const&);
-    void operator=(cusparseHandle const&);
-
-    cusparseHandle_t handle;
-};
-
-}
-
-
 #define CUSPARSE_CHECK(fn) do {                         \
         cusparseStatus_t _error = fn;                   \
         if (_error != CUSPARSE_STATUS_SUCCESS) {        \
@@ -52,8 +25,21 @@ class cusparseHandle
             snprintf(_err_msg, sizeof(_err_msg),        \
                      "CUSPARSE Error (%d): %s\n",       \
                      (int)(_error),                     \
-                     cusparse::errorString( _error));   \
+                     errorString( _error));             \
                                                         \
             AF_ERROR(_err_msg, AF_ERR_INTERNAL);        \
         }                                               \
     } while(0)
+
+class cusparseHandle : public common::MatrixAlgebraHandle<cusparseHandle, SparseHandle>
+{
+    public:
+        void createHandle(SparseHandle* handle);
+        void destroyHandle(SparseHandle handle) {
+            cusparseDestroy(handle);
+        }
+};
+
+typedef common::MatrixAlgebraHandle<cusparseHandle, SparseHandle> SparseHandleWrapper;
+
+}
diff --git a/src/backend/cuda/cusparseManager.cpp b/src/backend/cuda/cusparseManager.cpp
deleted file mode 100644
index bab949c8a7..0000000000
--- a/src/backend/cuda/cusparseManager.cpp
+++ /dev/null
@@ -1,54 +0,0 @@
-/*******************************************************
- * Copyright (c) 2014, ArrayFire
- * All rights reserved.
- *
- * This file is distributed under 3-clause BSD license.
- * The complete license agreement can be obtained at:
- * http://arrayfire.com/licenses/BSD-3-Clause
- ********************************************************/
-
-#include <cusparseManager.hpp>
-#include <platform.hpp>
-
-#include <stdexcept>
-#include <string>
-
-namespace cusparse
-{
-
-const char *errorString(cusparseStatus_t err)
-{
-    switch(err) {
-    case    CUSPARSE_STATUS_SUCCESS                     :   return "CUSPARSE_STATUS_SUCCESS"                    ;
-    case    CUSPARSE_STATUS_NOT_INITIALIZED             :   return "CUSPARSE_STATUS_NOT_INITIALIZED"            ;
-    case    CUSPARSE_STATUS_ALLOC_FAILED                :   return "CUSPARSE_STATUS_ALLOC_FAILED"               ;
-    case    CUSPARSE_STATUS_INVALID_VALUE               :   return "CUSPARSE_STATUS_INVALID_VALUE"              ;
-    case    CUSPARSE_STATUS_ARCH_MISMATCH               :   return "CUSPARSE_STATUS_ARCH_MISMATCH"              ;
-    case    CUSPARSE_STATUS_MAPPING_ERROR               :   return "CUSPARSE_STATUS_MAPPING_ERROR"              ;
-    case    CUSPARSE_STATUS_EXECUTION_FAILED            :   return "CUSPARSE_STATUS_EXECUTION_FAILED"           ;
-    case    CUSPARSE_STATUS_INTERNAL_ERROR              :   return "CUSPARSE_STATUS_INTERNAL_ERROR"             ;
-    case    CUSPARSE_STATUS_MATRIX_TYPE_NOT_SUPPORTED   :   return "CUSPARSE_STATUS_MATRIX_TYPE_NOT_SUPPORTED"  ;
-    case    CUSPARSE_STATUS_ZERO_PIVOT                  :   return "CUSPARSE_STATUS_ZERO_PIVOT"                 ;
-    default                                             :   return "UNKNOWN";
-    }
-}
-
-
-cusparseHandle::cusparseHandle()
-    : handle(0)
-{
-    CUSPARSE_CHECK(cusparseCreate(&handle));
-    CUSPARSE_CHECK(cusparseSetStream(handle, cuda::getStream(cuda::getActiveDeviceId())));
-}
-
-cusparseHandle::~cusparseHandle()
-{
-    cusparseDestroy(handle);
-}
-
-cusparseHandle_t cusparseHandle::get() const
-{
-    return handle;
-}
-
-}
diff --git a/src/backend/cuda/lu.cu b/src/backend/cuda/lu.cu
index 3051e0dade..79bbd5029e 100644
--- a/src/backend/cuda/lu.cu
+++ b/src/backend/cuda/lu.cu
@@ -129,7 +129,7 @@ Array<int> lu_inplace(Array<T> &in, const bool convert_pivot)
 
     int lwork = 0;
 
-    CUSOLVER_CHECK(getrf_buf_func<T>()(getcusolverDnHandle(),
+    CUSOLVER_CHECK(getrf_buf_func<T>()(cusolverDnHandle(),
                                        M, N,
                                        in.get(), in.strides()[1],
                                        &lwork));
@@ -137,7 +137,7 @@ Array<int> lu_inplace(Array<T> &in, const bool convert_pivot)
     T *workspace = memAlloc<T>(lwork);
     int *info = memAlloc<int>(1);
 
-    CUSOLVER_CHECK(getrf_func<T>()(getcusolverDnHandle(),
+    CUSOLVER_CHECK(getrf_func<T>()(cusolverDnHandle(),
                                    M, N,
                                    in.get(), in.strides()[1],
                                    workspace,
diff --git a/src/backend/cuda/platform.cpp b/src/backend/cuda/platform.cpp
index ab4949f154..3679d66ed6 100644
--- a/src/backend/cuda/platform.cpp
+++ b/src/backend/cuda/platform.cpp
@@ -424,26 +424,26 @@ FFTManager& cufftManager()
     return DeviceManager::getInstance().cufftManagers[getActiveDeviceId()];
 }
 
-cublasHandle_t getcublasHandle()
+BlasHandle cublasHandle()
 {
     DeviceManager& instance = DeviceManager::getInstance();
 
     int id = cuda::getActiveDeviceId();
 
     if (! instance.cublasHandles[id] )
-        instance.resetcublasHandle(id);
+        instance.cublasHandles[id].reset(new BlasHandleWrapper());
 
-    return instance.cublasHandles[id]->get();
+    return instance.cublasHandles[id].get()->get();
 }
 
-cusolverDnHandle_t getcusolverDnHandle()
+SolveHandle cusolverDnHandle()
 {
     DeviceManager& instance = DeviceManager::getInstance();
 
     int id = cuda::getActiveDeviceId();
 
     if (! instance.cusolverHandles[id] )
-        instance.resetcusolverHandle(id);
+        instance.cusolverHandles[id].reset(new SolveHandleWrapper());
 
     // FIXME
     // This is not an ideal case. It's just a hack.
@@ -460,19 +460,19 @@ cusolverDnHandle_t getcusolverDnHandle()
     //
     CUDA_CHECK(cudaStreamSynchronize(cuda::getStream(id)));
 
-    return instance.cusolverHandles[id]->get();
+    return instance.cusolverHandles[id].get()->get();
 }
 
-cusparseHandle_t getcusparseHandle()
+SparseHandle cusparseHandle()
 {
     DeviceManager& instance = DeviceManager::getInstance();
 
     int id = cuda::getActiveDeviceId();
 
     if (! instance.cusparseHandles[id] )
-        instance.resetcusparseHandle(id);
+        instance.cusparseHandles[id].reset(new SparseHandleWrapper());
 
-    return instance.cusparseHandles[id]->get();
+    return instance.cusparseHandles[id].get()->get();
 }
 
 DeviceManager::DeviceManager()
@@ -597,21 +597,6 @@ int DeviceManager::setActiveDevice(int device, int nId)
     return old;
 }
 
-void DeviceManager::resetcublasHandle(int device)
-{
-    cublasHandles[device].reset(new cublas::cublasHandle());
-}
-
-void DeviceManager::resetcusolverHandle(int device)
-{
-    cusolverHandles[device].reset(new cusolver::cusolverDnHandle());
-}
-
-void DeviceManager::resetcusparseHandle(int device)
-{
-    cusparseHandles[device].reset(new cusparse::cusparseHandle());
-}
-
 void sync(int device)
 {
     int currDevice = getActiveDeviceId();
diff --git a/src/backend/cuda/platform.hpp b/src/backend/cuda/platform.hpp
index a18e43b215..d665e9539b 100644
--- a/src/backend/cuda/platform.hpp
+++ b/src/backend/cuda/platform.hpp
@@ -17,9 +17,9 @@
 
 #include <GraphicsResourceManager.hpp>
 #include <cufft.hpp>
-#include <cublasManager.hpp>
-#include <cusolverDnManager.hpp>
-#include <cusparseManager.hpp>
+#include <cublas.hpp>
+#include <cusolverDn.hpp>
+#include <cusparse.hpp>
 
 namespace cuda
 {
@@ -84,11 +84,11 @@ GraphicsManager& interopManager();
 typedef common::FFTPlanCache<PlanCache, PlanType> FFTManager;
 FFTManager& cufftManager();
 
-cublasHandle_t getcublasHandle();
+BlasHandle cublasHandle();
 
-cusolverDnHandle_t getcusolverDnHandle();
+SolveHandle cusolverDnHandle();
 
-cusparseHandle_t getcusparseHandle();
+SparseHandle cusparseHandle();
 //
 ///////////////////////// END Sub-Managers /////////////////////
 
@@ -109,11 +109,11 @@ class DeviceManager
 
         friend FFTManager& cufftManager();
 
-        friend cublasHandle_t getcublasHandle();
+        friend BlasHandle cublasHandle();
 
-        friend cusolverDnHandle_t getcusolverDnHandle();
+        friend SolveHandle cusolverDnHandle();
 
-        friend cusparseHandle_t getcusparseHandle();
+        friend SparseHandle cusparseHandle();
 
         friend std::string getDeviceInfo(int device);
 
@@ -158,12 +158,6 @@ class DeviceManager
 
         int setActiveDevice(int device, int native = -1);
 
-        void resetcublasHandle(int device);
-
-        void resetcusolverHandle(int device);
-
-        void resetcusparseHandle(int device);
-
         int activeDev;
         int nDevices;
         cudaStream_t streams[MAX_DEVICES];
@@ -176,11 +170,11 @@ class DeviceManager
 
         FFTManager cufftManagers[MAX_DEVICES];
 
-        std::unique_ptr<cublas::cublasHandle> cublasHandles[MAX_DEVICES];
+        std::unique_ptr<BlasHandleWrapper> cublasHandles[MAX_DEVICES];
 
-        std::unique_ptr<cusolver::cusolverDnHandle> cusolverHandles[MAX_DEVICES];
+        std::unique_ptr<SolveHandleWrapper> cusolverHandles[MAX_DEVICES];
 
-        std::unique_ptr<cusparse::cusparseHandle> cusparseHandles[MAX_DEVICES];
+        std::unique_ptr<SparseHandleWrapper> cusparseHandles[MAX_DEVICES];
 };
 
 }
diff --git a/src/backend/cuda/qr.cu b/src/backend/cuda/qr.cu
index 1a46fd95f8..7a73512ea9 100644
--- a/src/backend/cuda/qr.cu
+++ b/src/backend/cuda/qr.cu
@@ -130,7 +130,7 @@ void qr(Array<T> &q, Array<T> &r, Array<T> &t, const Array<T> &in)
 
     int lwork = 0;
 
-    CUSOLVER_CHECK(geqrf_buf_func<T>()(getcusolverDnHandle(),
+    CUSOLVER_CHECK(geqrf_buf_func<T>()(cusolverDnHandle(),
                                        M, N,
                                        in_copy.get(), in_copy.strides()[1],
                                        &lwork));
@@ -140,7 +140,7 @@ void qr(Array<T> &q, Array<T> &r, Array<T> &t, const Array<T> &in)
     t = createEmptyArray<T>(af::dim4(min(M, N), 1, 1, 1));
     int *info = memAlloc<int>(1);
 
-    CUSOLVER_CHECK(geqrf_func<T>()(getcusolverDnHandle(),
+    CUSOLVER_CHECK(geqrf_func<T>()(cusolverDnHandle(),
                                    M, N,
                                    in_copy.get(), in_copy.strides()[1],
                                    t.get(),
@@ -157,7 +157,7 @@ void qr(Array<T> &q, Array<T> &r, Array<T> &t, const Array<T> &in)
     dim4 qdims(M, mn);
     q = identity<T>(qdims);
 
-    CUSOLVER_CHECK(mqr_func<T>()(getcusolverDnHandle(),
+    CUSOLVER_CHECK(mqr_func<T>()(cusolverDnHandle(),
                                  CUBLAS_SIDE_LEFT, CUBLAS_OP_N,
                                  q.dims()[0],
                                  q.dims()[1],
@@ -185,7 +185,7 @@ Array<T> qr_inplace(Array<T> &in)
 
     int lwork = 0;
 
-    CUSOLVER_CHECK(geqrf_buf_func<T>()(getcusolverDnHandle(),
+    CUSOLVER_CHECK(geqrf_buf_func<T>()(cusolverDnHandle(),
                                        M, N,
                                        in.get(), in.strides()[1],
                                        &lwork));
@@ -193,7 +193,7 @@ Array<T> qr_inplace(Array<T> &in)
     T *workspace = memAlloc<T>(lwork);
     int *info = memAlloc<int>(1);
 
-    CUSOLVER_CHECK(geqrf_func<T>()(getcusolverDnHandle(),
+    CUSOLVER_CHECK(geqrf_func<T>()(cusolverDnHandle(),
                                    M, N,
                                    in.get(), in.strides()[1],
                                    t.get(),
diff --git a/src/backend/cuda/solve.cu b/src/backend/cuda/solve.cu
index e8758fc2bd..c4984c4868 100644
--- a/src/backend/cuda/solve.cu
+++ b/src/backend/cuda/solve.cu
@@ -174,7 +174,7 @@ Array<T> solveLU(const Array<T> &A, const Array<int> &pivot,
 
     int *info = memAlloc<int>(1);
 
-    CUSOLVER_CHECK(getrs_func<T>()(getcusolverDnHandle(),
+    CUSOLVER_CHECK(getrs_func<T>()(cusolverDnHandle(),
                                    CUBLAS_OP_N,
                                    N, NRHS,
                                    A.get(), A.strides()[1],
@@ -199,7 +199,7 @@ Array<T> generalSolve(const Array<T> &a, const Array<T> &b)
 
     int *info = memAlloc<int>(1);
 
-    CUSOLVER_CHECK(getrs_func<T>()(getcusolverDnHandle(),
+    CUSOLVER_CHECK(getrs_func<T>()(cusolverDnHandle(),
                                    CUBLAS_OP_N,
                                    N, K,
                                    A.get(), A.strides()[1],
@@ -242,7 +242,7 @@ Array<T> leastSquares(const Array<T> &a, const Array<T> &b)
         int lwork = 0;
 
         // Get workspace needed for QR
-        CUSOLVER_CHECK(geqrf_solve_buf_func<T>()(getcusolverDnHandle(),
+        CUSOLVER_CHECK(geqrf_solve_buf_func<T>()(cusolverDnHandle(),
                                                  A.dims()[0], A.dims()[1],
                                                  A.get(), A.strides()[1],
                                                  &lwork));
@@ -252,7 +252,7 @@ Array<T> leastSquares(const Array<T> &a, const Array<T> &b)
         int *info = memAlloc<int>(1);
 
         // In place Perform in place QR
-        CUSOLVER_CHECK(geqrf_solve_func<T>()(getcusolverDnHandle(),
+        CUSOLVER_CHECK(geqrf_solve_func<T>()(cusolverDnHandle(),
                                              A.dims()[0], A.dims()[1],
                                              A.get(), A.strides()[1],
                                              t.get(),
@@ -270,7 +270,7 @@ Array<T> leastSquares(const Array<T> &a, const Array<T> &b)
         B.resetDims(dim4(N, K));
 
         // matmul(Q, Bpad)
-        CUSOLVER_CHECK(mqr_solve_func<T>()(getcusolverDnHandle(),
+        CUSOLVER_CHECK(mqr_solve_func<T>()(cusolverDnHandle(),
                                            CUBLAS_SIDE_LEFT, CUBLAS_OP_N,
                                            B.dims()[0],
                                            B.dims()[1],
@@ -300,7 +300,7 @@ Array<T> leastSquares(const Array<T> &a, const Array<T> &b)
         int lwork = 0;
 
         // Get workspace needed for QR
-        CUSOLVER_CHECK(geqrf_solve_buf_func<T>()(getcusolverDnHandle(),
+        CUSOLVER_CHECK(geqrf_solve_buf_func<T>()(cusolverDnHandle(),
                                                  A.dims()[0], A.dims()[1],
                                                  A.get(), A.strides()[1],
                                                  &lwork));
@@ -310,7 +310,7 @@ Array<T> leastSquares(const Array<T> &a, const Array<T> &b)
         int *info = memAlloc<int>(1);
 
         // In place Perform in place QR
-        CUSOLVER_CHECK(geqrf_solve_func<T>()(getcusolverDnHandle(),
+        CUSOLVER_CHECK(geqrf_solve_func<T>()(cusolverDnHandle(),
                                              A.dims()[0], A.dims()[1],
                                              A.get(), A.strides()[1],
                                              t.get(),
@@ -318,7 +318,7 @@ Array<T> leastSquares(const Array<T> &a, const Array<T> &b)
                                              info));
 
         // matmul(Q1, B)
-        CUSOLVER_CHECK(mqr_solve_func<T>()(getcusolverDnHandle(),
+        CUSOLVER_CHECK(mqr_solve_func<T>()(cusolverDnHandle(),
                                            CUBLAS_SIDE_LEFT,
                                            trans<T>(),
                                            M, K, N,
diff --git a/src/backend/cuda/sparse.cu b/src/backend/cuda/sparse.cu
index 7c16226a50..6ce7c9226b 100644
--- a/src/backend/cuda/sparse.cu
+++ b/src/backend/cuda/sparse.cu
@@ -261,7 +261,7 @@ SparseArray<T> sparseConvertDenseToStorage(const Array<T> &in)
 
     int nNZ = -1;
     CUSPARSE_CHECK(nnz_func<T>()(
-                        getcusparseHandle(),
+                        cusparseHandle(),
                         dir,
                         M, N,
                         descr,
@@ -282,7 +282,7 @@ SparseArray<T> sparseConvertDenseToStorage(const Array<T> &in)
 
     if(stype == AF_STORAGE_CSR)
         CUSPARSE_CHECK(dense2csr_func<T>()(
-                        getcusparseHandle(),
+                        cusparseHandle(),
                         M, N,
                         descr,
                         in.get(), in.strides()[1],
@@ -290,7 +290,7 @@ SparseArray<T> sparseConvertDenseToStorage(const Array<T> &in)
                         values.get(), rowIdx.get(), colIdx.get()));
     else
         CUSPARSE_CHECK(dense2csc_func<T>()(
-                        getcusparseHandle(),
+                        cusparseHandle(),
                         M, N,
                         descr,
                         in.get(), in.strides()[1],
@@ -336,7 +336,7 @@ Array<T> sparseConvertStorageToDense(const SparseArray<T> &in)
 
     if(stype == AF_STORAGE_CSR)
         CUSPARSE_CHECK(csr2dense_func<T>()(
-                        getcusparseHandle(),
+                        cusparseHandle(),
                         M, N,
                         descr,
                         in.getValues().get(),
@@ -345,7 +345,7 @@ Array<T> sparseConvertStorageToDense(const SparseArray<T> &in)
                         dense.get(), d_strides1));
     else
         CUSPARSE_CHECK(csc2dense_func<T>()(
-                        getcusparseHandle(),
+                        cusparseHandle(),
                         M, N,
                         descr,
                         in.getValues().get(),
@@ -377,7 +377,7 @@ SparseArray<T> sparseConvertStorageToStorage(const SparseArray<T> &in)
 
         // cusparse function to expand compressed row into coordinate
         CUSPARSE_CHECK(cusparseXcsr2coo(
-                        getcusparseHandle(),
+                        cusparseHandle(),
                         in.getRowIdx().get(),
                         nNZ, in.dims()[0],
                         converted.getRowIdx().get(),
@@ -386,23 +386,23 @@ SparseArray<T> sparseConvertStorageToStorage(const SparseArray<T> &in)
         // Call sort
         size_t pBufferSizeInBytes = 0;
         CUSPARSE_CHECK(cusparseXcoosort_bufferSizeExt(
-                        getcusparseHandle(),
+                        cusparseHandle(),
                         in.dims()[0], in.dims()[1], nNZ,
                         converted.getRowIdx().get(), converted.getColIdx().get(),
                         &pBufferSizeInBytes));
         shared_ptr<char> pBuffer(memAlloc<char>(pBufferSizeInBytes), memFree<char>);
 
         shared_ptr<int> P(memAlloc<int>(nNZ), memFree<int>);
-        CUSPARSE_CHECK(cusparseCreateIdentityPermutation(getcusparseHandle(), nNZ, P.get()));
+        CUSPARSE_CHECK(cusparseCreateIdentityPermutation(cusparseHandle(), nNZ, P.get()));
 
         CUSPARSE_CHECK(cusparseXcoosortByColumn(
-                       getcusparseHandle(),
+                       cusparseHandle(),
                        in.dims()[0], in.dims()[1], nNZ,
                        converted.getRowIdx().get(), converted.getColIdx().get(),
                        P.get(), (void*)pBuffer.get()));
 
         CUSPARSE_CHECK(gthr_func<T>()(
-                       getcusparseHandle(), nNZ,
+                       cusparseHandle(), nNZ,
                        in.getValues().get(),
                        converted.getValues().get(),
                        P.get(), CUSPARSE_INDEX_BASE_ZERO));
@@ -421,23 +421,23 @@ SparseArray<T> sparseConvertStorageToStorage(const SparseArray<T> &in)
         {
             size_t pBufferSizeInBytes = 0;
             CUSPARSE_CHECK(cusparseXcoosort_bufferSizeExt(
-                            getcusparseHandle(),
+                            cusparseHandle(),
                             cooT.dims()[0], cooT.dims()[1], nNZ,
                             cooT.getRowIdx().get(), cooT.getColIdx().get(),
                             &pBufferSizeInBytes));
             shared_ptr<char> pBuffer(memAlloc<char>(pBufferSizeInBytes), memFree<char>);
 
             shared_ptr<int> P(memAlloc<int>(nNZ), memFree<int>);
-            CUSPARSE_CHECK(cusparseCreateIdentityPermutation(getcusparseHandle(), nNZ, P.get()));
+            CUSPARSE_CHECK(cusparseCreateIdentityPermutation(cusparseHandle(), nNZ, P.get()));
 
             CUSPARSE_CHECK(cusparseXcoosortByRow(
-                           getcusparseHandle(),
+                           cusparseHandle(),
                            cooT.dims()[0], cooT.dims()[1], nNZ,
                            cooT.getRowIdx().get(), cooT.getColIdx().get(),
                            P.get(), (void*)pBuffer.get()));
 
             CUSPARSE_CHECK(gthr_func<T>()(
-                           getcusparseHandle(), nNZ,
+                           cusparseHandle(), nNZ,
                            in.getValues().get(),
                            cooT.getValues().get(),
                            P.get(), CUSPARSE_INDEX_BASE_ZERO));
@@ -456,7 +456,7 @@ SparseArray<T> sparseConvertStorageToStorage(const SparseArray<T> &in)
 
         // cusparse function to compress row from coordinate
         CUSPARSE_CHECK(cusparseXcoo2csr(
-                        getcusparseHandle(),
+                        cusparseHandle(),
                         cooT.getRowIdx().get(),
                         nNZ, cooT.dims()[0],
                         converted.getRowIdx().get(),
diff --git a/src/backend/cuda/sparse_blas.cpp b/src/backend/cuda/sparse_blas.cpp
index 1d6dc27775..be768d054d 100644
--- a/src/backend/cuda/sparse_blas.cpp
+++ b/src/backend/cuda/sparse_blas.cpp
@@ -155,7 +155,7 @@ Array<T> matmul(const common::SparseArray<T> lhs, const Array<T> rhs,
     // and not OP(A) (gemm wants row/col of OP(A)).
     if(rDims[rColDim] == 1) {
         CUSPARSE_CHECK(csrmv_func<T>()(
-                       getcusparseHandle(),
+                       cusparseHandle(),
                        lOpts,
                        lDims[0], lDims[1], lhs.getNNZ(),
                        &alpha,
@@ -166,7 +166,7 @@ Array<T> matmul(const common::SparseArray<T> lhs, const Array<T> rhs,
                        out.get()));
     } else {
         CUSPARSE_CHECK(csrmm_func<T>()(
-                       getcusparseHandle(),
+                       cusparseHandle(),
                        lOpts,
                        lDims[0], rDims[rColDim], lDims[1], lhs.getNNZ(),
                        &alpha,
diff --git a/src/backend/cuda/svd.cu b/src/backend/cuda/svd.cu
index 62a016b019..9645aecf66 100644
--- a/src/backend/cuda/svd.cu
+++ b/src/backend/cuda/svd.cu
@@ -17,7 +17,7 @@
 #include <math.hpp>
 #include <err_common.hpp>
 
-#include <cusolverDnManager.hpp>
+#include <cusolverDn.hpp>
 
 namespace cuda
 {
@@ -86,14 +86,14 @@ SVD_SPECIALIZE(cdouble, double, Z);
 
         int lwork = 0;
 
-        CUSOLVER_CHECK(gesvd_buf_func<T>(getcusolverDnHandle(), M, N, &lwork));
+        CUSOLVER_CHECK(gesvd_buf_func<T>(cusolverDnHandle(), M, N, &lwork));
 
         T  *lWorkspace = memAlloc<T >(lwork);
         Tr *rWorkspace = memAlloc<Tr>(5 * std::min(M, N));
 
         int *info = memAlloc<int>(1);
 
-        gesvd_func<T, Tr>(getcusolverDnHandle(), 'A', 'A', M, N, in.get(),
+        gesvd_func<T, Tr>(cusolverDnHandle(), 'A', 'A', M, N, in.get(),
                           M, s.get(), u.get(), M, vt.get(), N,
                           lWorkspace, lwork, rWorkspace, info);
 

From c27933d4aec838aecdd43989a6335a2c53aef145 Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Sat, 14 Jan 2017 17:13:46 +0530
Subject: [PATCH 22/82] Remove obsolete files

Forgot to remove these files in earlier commit
---
 src/backend/cuda/interopManager.cpp | 171 ----------------------------
 src/backend/cuda/interopManager.hpp |  71 ------------
 2 files changed, 242 deletions(-)
 delete mode 100644 src/backend/cuda/interopManager.cpp
 delete mode 100644 src/backend/cuda/interopManager.hpp

diff --git a/src/backend/cuda/interopManager.cpp b/src/backend/cuda/interopManager.cpp
deleted file mode 100644
index a8339a52f4..0000000000
--- a/src/backend/cuda/interopManager.cpp
+++ /dev/null
@@ -1,171 +0,0 @@
-/*******************************************************
- * Copyright (c) 2014, ArrayFire
- * All rights reserved.
- *
- * This file is distributed under 3-clause BSD license.
- * The complete license agreement can be obtained at:
- * http://arrayfire.com/licenses/BSD-3-Clause
- ********************************************************/
-
-// Parts of this code sourced from SnopyDogy
-// https://gist.github.com/SnopyDogy/a9a22497a893ec86aa3e
-
-#if defined(WITH_GRAPHICS)
-
-#include <interopManager.hpp>
-#include <err_cuda.hpp>
-#include <util.hpp>
-#include <cstdio>
-
-namespace cuda
-{
-
-void InteropManager::destroyResources()
-{
-    typedef std::vector<CGR_t>::iterator CGRIter_t;
-
-    int n = getActiveDeviceId();
-    for(iter_t iter = interop_maps[n].begin(); iter != interop_maps[n].end(); iter++) {
-        for(CGRIter_t ct = (iter->second).begin(); ct != (iter->second).end(); ct++) {
-            CUDA_CHECK(cudaGraphicsUnregisterResource(*ct));
-        }
-        (iter->second).clear();
-    }
-}
-
-InteropManager::~InteropManager()
-{
-    try {
-        for(int i = 0; i < getDeviceCount(); i++) {
-            setDevice(i);
-            destroyResources();
-        }
-    } catch (AfError &ex) {
-
-        std::string perr = getEnvVar("AF_PRINT_ERRORS");
-        if(!perr.empty()) {
-            if(perr != "0")
-                fprintf(stderr, "%s\n", ex.what());
-        }
-    }
-}
-
-interop_t& InteropManager::getDeviceMap(int device)
-{
-    return (device == -1) ? interop_maps[getActiveDeviceId()] : interop_maps[device];
-}
-
-CGR_t* InteropManager::getBufferResource(const forge::Image* key)
-{
-    void* key_value = (void*)key;
-    interop_t& i_map = getDeviceMap();
-
-    if(i_map.find(key_value) == i_map.end()) {
-        CGR_t pixelsResource;
-        // Register pixels with CUDA
-        CUDA_CHECK(cudaGraphicsGLRegisterBuffer(&pixelsResource, key->pixels(), cudaGraphicsMapFlagsWriteDiscard));
-        // TODO:
-        // A way to store multiple buffers and take PBO/CBO etc as
-        // argument and return the appropriate buffer
-        std::vector<CGR_t> vec(1);
-        vec[0] = pixelsResource;
-        i_map[key_value] = vec;
-    }
-
-    return &i_map[key_value].front();
-}
-
-CGR_t* InteropManager::getBufferResource(const forge::Plot* key)
-{
-    void* key_value = (void*)key;
-    interop_t& i_map = getDeviceMap();
-
-    iter_t iter = i_map.find(key_value);
-
-    if(iter == i_map.end()) {
-        CGR_t vboResource;
-        // Register VBO with CUDA
-        CUDA_CHECK(cudaGraphicsGLRegisterBuffer(&vboResource, key->vertices(), cudaGraphicsMapFlagsWriteDiscard));
-        // TODO:
-        // A way to store multiple buffers and take PBO/CBO etc as
-        // argument and return the appropriate buffer
-        std::vector<CGR_t> vec(1);
-        vec[0] = vboResource;
-        i_map[key_value] = vec;
-    }
-
-    return &i_map[key_value].front();
-}
-
-CGR_t* InteropManager::getBufferResource(const forge::Histogram* key)
-{
-    void* key_value = (void*)key;
-    interop_t& i_map = getDeviceMap();
-
-    iter_t iter = i_map.find(key_value);
-
-    if(iter == i_map.end()) {
-        CGR_t vboResource;
-        // Register VBO with CUDA
-        CUDA_CHECK(cudaGraphicsGLRegisterBuffer(&vboResource, key->vertices(), cudaGraphicsMapFlagsWriteDiscard));
-        // TODO:
-        // A way to store multiple buffers and take PBO/CBO etc as
-        // argument and return the appropriate buffer
-        std::vector<CGR_t> vec(1);
-        vec[0] = vboResource;
-        i_map[key_value] = vec;
-    }
-
-    return &i_map[key_value].front();
-}
-
-CGR_t* InteropManager::getBufferResource(const forge::Surface* key)
-{
-    void* key_value = (void*)key;
-    interop_t& i_map = getDeviceMap();
-
-    iter_t iter = i_map.find(key_value);
-
-    if(iter == i_map.end()) {
-        CGR_t  vboResource;
-        // Register VBO with CUDA
-        CUDA_CHECK(cudaGraphicsGLRegisterBuffer(&vboResource, key->vertices(), cudaGraphicsMapFlagsWriteDiscard));
-        // TODO:
-        // A way to store multiple buffers and take PBO/CBO etc as
-        // argument and return the appropriate buffer
-        std::vector<CGR_t> vec(1);
-        vec[0] = vboResource;
-        i_map[key_value] = vec;
-    }
-
-    return &i_map[key_value].front();
-}
-
-CGR_t* InteropManager::getBufferResource(const forge::VectorField* key)
-{
-    void* key_value = (void*)key;
-    interop_t& i_map = getDeviceMap();
-
-    iter_t iter = i_map.find(key_value);
-
-    if(iter == i_map.end()) {
-        CGR_t   pResource;
-        CGR_t   dResource;
-        // Register VBO with CUDA
-        CUDA_CHECK(cudaGraphicsGLRegisterBuffer(&pResource, key->vertices(), cudaGraphicsMapFlagsWriteDiscard));
-        CUDA_CHECK(cudaGraphicsGLRegisterBuffer(&dResource, key->directions(), cudaGraphicsMapFlagsWriteDiscard));
-        // TODO:
-        // A way to store multiple buffers and take PBO/CBO etc as
-        // argument and return the appropriate buffer
-        std::vector<CGR_t> vec(2);
-        vec[0] = pResource;
-        vec[1] = dResource;
-        i_map[key_value] = vec;
-    }
-
-    return &i_map[key_value].front();
-}
-
-}
-
-#endif
diff --git a/src/backend/cuda/interopManager.hpp b/src/backend/cuda/interopManager.hpp
deleted file mode 100644
index 6864079ee9..0000000000
--- a/src/backend/cuda/interopManager.hpp
+++ /dev/null
@@ -1,71 +0,0 @@
-/*******************************************************
- * Copyright (c) 2014, ArrayFire
- * All rights reserved.
- *
- * This file is distributed under 3-clause BSD license.
- * The complete license agreement can be obtained at:
- * http://arrayfire.com/licenses/BSD-3-Clause
- ********************************************************/
-
-// Parts of this code sourced from SnopyDogy
-// https://gist.github.com/SnopyDogy/a9a22497a893ec86aa3e
-
-#if defined(WITH_GRAPHICS)
-
-#if defined(OS_WIN)
-#include <windows.h>
-#endif
-
-// cuda_gl_interop.h does not include OpenGL headers for ARM
-#include <glbinding/gl/gl.h>
-#include <glbinding/Binding.h>
-
-#if defined(__arm__) || defined(__aarch64__)
-using namespace gl;
-#define GL_VERSION gl::GL_VERSION
-#endif
-
-#include <platform.hpp>
-#include <graphics_common.hpp>
-
-#include <cuda.h>
-#include <cuda_runtime.h>
-#include <cuda_gl_interop.h>
-
-#include <map>
-#include <vector>
-
-using af::dim4;
-
-namespace cuda
-{
-
-typedef std::map<void *, std::vector<cudaGraphicsResource_t> > interop_t;
-typedef interop_t::iterator iter_t;
-typedef cudaGraphicsResource_t CGR_t;
-
-// Manager Class for cudaPBOResource: calls garbage collection at the end of the program
-class InteropManager
-{
-    private:
-        interop_t interop_maps[DeviceManager::MAX_DEVICES];
-
-    public:
-        InteropManager() {}
-        ~InteropManager();
-        CGR_t* getBufferResource(const forge::Image       *handle);
-        CGR_t* getBufferResource(const forge::Plot        *handle);
-        CGR_t* getBufferResource(const forge::Histogram   *handle);
-        CGR_t* getBufferResource(const forge::Surface     *handle);
-        CGR_t* getBufferResource(const forge::VectorField *handle);
-
-    protected:
-        InteropManager(InteropManager const&);
-        void operator=(InteropManager const&);
-        interop_t& getDeviceMap(int device = -1); // default will return current device
-        void destroyResources();
-};
-
-}
-
-#endif

From 8387dad5028454088d226186404a027512d3b253 Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Sat, 14 Jan 2017 17:24:18 +0530
Subject: [PATCH 23/82] Remove unnecessary std::cout

---
 src/backend/cpu/platform.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/backend/cpu/platform.cpp b/src/backend/cpu/platform.cpp
index 105346ea37..c8103bb93f 100644
--- a/src/backend/cpu/platform.cpp
+++ b/src/backend/cpu/platform.cpp
@@ -99,7 +99,6 @@ CPUInfo::CPUInfo()
         }
     } else {
         mVendorId = "Unkown";
-        std::cout<< "Unexpected vendor id" << std::endl;
     }
     // Get processor brand string
     // This seems to be working for both Intel & AMD vendors

From c7b07a0639a6911507bcab8841ea2a3c736b58d3 Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Sat, 14 Jan 2017 17:28:46 +0530
Subject: [PATCH 24/82] Remove checks for CUDA versions less than 7.0

Minimum required CUDA version currently is 7.0, hence the checks
are not needed any more.
---
 src/backend/cuda/cublas.cpp | 2 --
 src/backend/cuda/cufft.cpp  | 3 +--
 2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/src/backend/cuda/cublas.cpp b/src/backend/cuda/cublas.cpp
index 5e40781c0a..685c7d26f6 100644
--- a/src/backend/cuda/cublas.cpp
+++ b/src/backend/cuda/cublas.cpp
@@ -25,9 +25,7 @@ const char *errorString(cublasStatus_t err)
     case CUBLAS_STATUS_MAPPING_ERROR   : return "CUBLAS_STATUS_MAPPING_ERROR"   ;
     case CUBLAS_STATUS_EXECUTION_FAILED: return "CUBLAS_STATUS_EXECUTION_FAILED";
     case CUBLAS_STATUS_INTERNAL_ERROR  : return "CUBLAS_STATUS_INTERNAL_ERROR"  ;
-#if CUDA_VERSION > 5050
     case CUBLAS_STATUS_NOT_SUPPORTED   : return "CUBLAS_STATUS_NOT_SUPPORTED"   ;
-#endif
     default: return "UNKNOWN";
     }
 }
diff --git a/src/backend/cuda/cufft.cpp b/src/backend/cuda/cufft.cpp
index f8d36e7ee8..dcb1cee5bd 100644
--- a/src/backend/cuda/cufft.cpp
+++ b/src/backend/cuda/cufft.cpp
@@ -60,13 +60,12 @@ const char * _cufftGetResultString(cufftResult res)
         case CUFFT_NO_WORKSPACE:
             return "cuFFT: no workspace provided";
 
-#if CUDA_VERSION >= 6050
         case CUFFT_NOT_IMPLEMENTED:
             return "cuFFT: not implemented";
 
         case CUFFT_LICENSE_ERROR:
             return "cuFFT: license error";
-#endif
+
 #if CUDA_VERSION >= 8000
         case CUFFT_NOT_SUPPORTED:
             return "cuFFT: not supported";

From 25b7d90f4313fce8c52d1d47053256023c321356 Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Sat, 14 Jan 2017 20:03:06 +0530
Subject: [PATCH 25/82] Add new getActiveStream fn in cuda backend

This abbrievates the lengthy fn call getStream(getActiveDeviceId())
---
 src/backend/cuda/Array.cpp                    | 19 ++++-----
 src/backend/cuda/copy.cu                      |  4 +-
 src/backend/cuda/cublas.cpp                   |  2 +-
 src/backend/cuda/cufft.cpp                    |  2 +-
 src/backend/cuda/cusparse.cpp                 |  2 +-
 src/backend/cuda/debug_cuda.hpp               | 12 +++---
 src/backend/cuda/hist_graphics.cpp            |  6 +--
 src/backend/cuda/image.cpp                    |  6 +--
 src/backend/cuda/jit.cpp                      |  2 +-
 src/backend/cuda/kernel/convolve.cu           |  6 +--
 src/backend/cuda/kernel/convolve_separable.cu |  2 +-
 src/backend/cuda/kernel/fast.hpp              |  6 +--
 src/backend/cuda/kernel/harris.hpp            | 14 +++----
 src/backend/cuda/kernel/homography.hpp        | 20 ++++-----
 src/backend/cuda/kernel/ireduce.hpp           | 10 ++---
 src/backend/cuda/kernel/orb.hpp               | 20 ++++-----
 src/backend/cuda/kernel/reduce.hpp            |  8 ++--
 src/backend/cuda/kernel/regions.hpp           |  8 ++--
 src/backend/cuda/kernel/sift_nonfree.hpp      | 42 +++++++++----------
 src/backend/cuda/kernel/sort.hpp              |  2 +-
 src/backend/cuda/kernel/sort_by_key.hpp       |  4 +-
 src/backend/cuda/kernel/susan.hpp             |  6 +--
 src/backend/cuda/kernel/transform.hpp         |  2 +-
 src/backend/cuda/kernel/where.hpp             |  4 +-
 src/backend/cuda/morph3d_impl.hpp             |  2 +-
 src/backend/cuda/morph_impl.hpp               |  2 +-
 src/backend/cuda/platform.cpp                 |  7 +++-
 src/backend/cuda/platform.hpp                 |  2 +
 src/backend/cuda/plot.cpp                     |  6 +--
 src/backend/cuda/sparse.cu                    |  6 +--
 src/backend/cuda/surface.cpp                  |  6 +--
 src/backend/cuda/vector_field.cpp             |  8 ++--
 32 files changed, 126 insertions(+), 122 deletions(-)

diff --git a/src/backend/cuda/Array.cpp b/src/backend/cuda/Array.cpp
index 90b9f8d8d1..e94bcb787a 100644
--- a/src/backend/cuda/Array.cpp
+++ b/src/backend/cuda/Array.cpp
@@ -58,12 +58,12 @@ namespace cuda
 #endif
         if (!is_device) {
             CUDA_CHECK(cudaMemcpyAsync(data.get(), in_data, dims.elements() * sizeof(T),
-                                       cudaMemcpyHostToDevice, cuda::getStream(cuda::getActiveDeviceId())));
-            CUDA_CHECK(cudaStreamSynchronize(cuda::getStream(cuda::getActiveDeviceId())));
+                                       cudaMemcpyHostToDevice, cuda::getActiveStream()));
+            CUDA_CHECK(cudaStreamSynchronize(cuda::getActiveStream()));
         } else if (copy_device) {
             CUDA_CHECK(cudaMemcpyAsync(data.get(), in_data, dims.elements() * sizeof(T),
-                                       cudaMemcpyDeviceToDevice, cuda::getStream(cuda::getActiveDeviceId())));
-            CUDA_CHECK(cudaStreamSynchronize(cuda::getStream(cuda::getActiveDeviceId())));
+                                       cudaMemcpyDeviceToDevice, cuda::getActiveStream()));
+            CUDA_CHECK(cudaStreamSynchronize(cuda::getActiveStream()));
         }
     }
 
@@ -107,7 +107,7 @@ namespace cuda
         owner(true)
     {
         if (!is_device) {
-            cudaStream_t stream = getStream(getActiveDeviceId());
+            cudaStream_t stream = getActiveStream();
             CUDA_CHECK(cudaMemcpyAsync(data.get(), in_data, info.total() * sizeof(T),
                                        cudaMemcpyHostToDevice, stream));
             CUDA_CHECK(cudaStreamSynchronize(stream));
@@ -340,9 +340,8 @@ namespace cuda
 
         T *ptr = arr.get();
 
-        CUDA_CHECK(cudaMemcpyAsync(ptr, data, bytes, cudaMemcpyHostToDevice,
-                    cuda::getStream(cuda::getActiveDeviceId())));
-        CUDA_CHECK(cudaStreamSynchronize(cuda::getStream(cuda::getActiveDeviceId())));
+        CUDA_CHECK(cudaMemcpyAsync(ptr, data, bytes, cudaMemcpyHostToDevice, cuda::getActiveStream()));
+        CUDA_CHECK(cudaStreamSynchronize(cuda::getActiveStream()));
 
         return;
     }
@@ -357,9 +356,7 @@ namespace cuda
 
         T *ptr = arr.get();
 
-        CUDA_CHECK(cudaMemcpyAsync(ptr, data,
-                                   bytes, cudaMemcpyDeviceToDevice,
-                                   cuda::getStream(cuda::getActiveDeviceId())));
+        CUDA_CHECK(cudaMemcpyAsync(ptr, data, bytes, cudaMemcpyDeviceToDevice, cuda::getActiveStream()));
 
         return;
     }
diff --git a/src/backend/cuda/copy.cu b/src/backend/cuda/copy.cu
index 7164d63635..6a92051490 100644
--- a/src/backend/cuda/copy.cu
+++ b/src/backend/cuda/copy.cu
@@ -54,7 +54,7 @@ namespace cuda
             CUDA_CHECK(cudaMemcpyAsync(out.get(), A.get(),
                                        A.elements() * sizeof(T),
                                        cudaMemcpyDeviceToDevice,
-                                       cuda::getStream(cuda::getActiveDeviceId())));
+                                       cuda::getActiveStream()));
         } else {
             // FIXME: Seems to fail when using Param<T>
             kernel::memcopy(out.get(), out.strides().get(), A.get(), A.dims().get(),
@@ -97,7 +97,7 @@ namespace cuda
                 CUDA_CHECK(cudaMemcpyAsync(out.get(), in.get(),
                                            in.elements() * sizeof(T),
                                            cudaMemcpyDeviceToDevice,
-                                           cuda::getStream(cuda::getActiveDeviceId())));
+                                           cuda::getActiveStream()));
             } else {
                 kernel::copy<T, T>(out, in, in.ndims(), scalar<T>(0), 1);
             }
diff --git a/src/backend/cuda/cublas.cpp b/src/backend/cuda/cublas.cpp
index 685c7d26f6..515a6e4bd0 100644
--- a/src/backend/cuda/cublas.cpp
+++ b/src/backend/cuda/cublas.cpp
@@ -33,6 +33,6 @@ const char *errorString(cublasStatus_t err)
 void cublasHandle::createHandle(BlasHandle* handle)
 {
     CUBLAS_CHECK(cublasCreate(handle));
-    CUBLAS_CHECK(cublasSetStream(*handle, cuda::getStream(cuda::getActiveDeviceId())));
+    CUBLAS_CHECK(cublasSetStream(*handle, cuda::getActiveStream()));
 }
 }
diff --git a/src/backend/cuda/cufft.cpp b/src/backend/cuda/cufft.cpp
index dcb1cee5bd..ac743f47f8 100644
--- a/src/backend/cuda/cufft.cpp
+++ b/src/backend/cuda/cufft.cpp
@@ -134,7 +134,7 @@ PlanType findPlan(int rank, int *n,
                                   type, batch));
     }
 
-    cufftSetStream(retVal, cuda::getStream(cuda::getActiveDeviceId()));
+    cufftSetStream(retVal, cuda::getActiveStream());
 
     // push the plan into plan cache
     planner.push(key_string, retVal);
diff --git a/src/backend/cuda/cusparse.cpp b/src/backend/cuda/cusparse.cpp
index 08204f51b7..526c1c50a3 100644
--- a/src/backend/cuda/cusparse.cpp
+++ b/src/backend/cuda/cusparse.cpp
@@ -34,6 +34,6 @@ const char *errorString(cusparseStatus_t err)
 void cusparseHandle::createHandle(SparseHandle* handle)
 {
     CUSPARSE_CHECK(cusparseCreate(handle));
-    CUSPARSE_CHECK(cusparseSetStream(*handle, cuda::getStream(cuda::getActiveDeviceId())));
+    CUSPARSE_CHECK(cusparseSetStream(*handle, cuda::getActiveStream()));
 }
 }
diff --git a/src/backend/cuda/debug_cuda.hpp b/src/backend/cuda/debug_cuda.hpp
index f5424950dc..f7c60d37b2 100644
--- a/src/backend/cuda/debug_cuda.hpp
+++ b/src/backend/cuda/debug_cuda.hpp
@@ -13,7 +13,7 @@
 #include <thrust/version.h>
 #include <thrust/system/cuda/detail/par.h>
 
-#define THRUST_STREAM thrust::cuda::par.on(cuda::getStream(cuda::getActiveDeviceId()))
+#define THRUST_STREAM thrust::cuda::par.on(cuda::getActiveStream())
 
 #if THRUST_MAJOR_VERSION>=1 && THRUST_MINOR_VERSION>=8
 
@@ -24,20 +24,20 @@
 
 #define THRUST_SELECT(fn, ...) \
     do {                          \
-        CUDA_CHECK(cudaStreamSynchronize(cuda::getStream(cuda::getActiveDeviceId()))); \
+        CUDA_CHECK(cudaStreamSynchronize(cuda::getActiveStream())); \
         fn(__VA_ARGS__);       \
     } while(0)
 
 #define THRUST_SELECT_OUT(res, fn, ...) \
     do {                          \
-        CUDA_CHECK(cudaStreamSynchronize(cuda::getStream(cuda::getActiveDeviceId()))); \
+        CUDA_CHECK(cudaStreamSynchronize(cuda::getActiveStream())); \
         res = fn(__VA_ARGS__);       \
     } while(0)
 
 #endif
 
 #define CUDA_LAUNCH_SMEM(fn, blks, thrds, smem_size, ...) \
-	fn<<<blks, thrds, smem_size, cuda::getStream(cuda::getActiveDeviceId())>>>(__VA_ARGS__)
+	fn<<<blks, thrds, smem_size, cuda::getActiveStream()>>>(__VA_ARGS__)
 
 #define CUDA_LAUNCH(fn, blks, thrds, ...) \
 	CUDA_LAUNCH_SMEM(fn, blks, thrds, 0, __VA_ARGS__)
@@ -46,14 +46,14 @@
 #ifndef NDEBUG
 
 #define POST_LAUNCH_CHECK() do {                        \
-        CUDA_CHECK(cudaStreamSynchronize(cuda::getStream(cuda::getActiveDeviceId()))); \
+        CUDA_CHECK(cudaStreamSynchronize(cuda::getActiveStream())); \
     } while(0)                                          \
 
 #else
 
 #define POST_LAUNCH_CHECK() do {                                        \
     if(cuda::synchronize_calls()) {                                     \
-        CUDA_CHECK(cudaStreamSynchronize(cuda::getStream(cuda::getActiveDeviceId()))); \
+        CUDA_CHECK(cudaStreamSynchronize(cuda::getActiveStream())); \
     } else {                                                            \
         CUDA_CHECK(cudaPeekAtLastError());                              \
     }                                                                   \
diff --git a/src/backend/cuda/hist_graphics.cpp b/src/backend/cuda/hist_graphics.cpp
index 52c959b4d4..f74e03c23a 100644
--- a/src/backend/cuda/hist_graphics.cpp
+++ b/src/backend/cuda/hist_graphics.cpp
@@ -31,11 +31,11 @@ void copy_histogram(const Array<T> &data, const forge::Histogram* hist)
         // Map resource. Copy data to VBO. Unmap resource.
         size_t num_bytes = hist->verticesSize();
         T* d_vbo = NULL;
-        cudaGraphicsMapResources(1, resources, cuda::getStream(cuda::getActiveDeviceId()));
+        cudaGraphicsMapResources(1, resources, cuda::getActiveStream());
         cudaGraphicsResourceGetMappedPointer((void **)&d_vbo, &num_bytes, resources[0]);
         cudaMemcpyAsync(d_vbo, d_P, num_bytes, cudaMemcpyDeviceToDevice,
-                        cuda::getStream(cuda::getActiveDeviceId()));
-        cudaGraphicsUnmapResources(1, resources, cuda::getStream(cuda::getActiveDeviceId()));
+                        cuda::getActiveStream());
+        cudaGraphicsUnmapResources(1, resources, cuda::getActiveStream());
 
         CheckGL("After cuda resource copy");
 
diff --git a/src/backend/cuda/image.cpp b/src/backend/cuda/image.cpp
index 816cc2b8cd..de9d0efa9c 100644
--- a/src/backend/cuda/image.cpp
+++ b/src/backend/cuda/image.cpp
@@ -36,11 +36,11 @@ void copy_image(const Array<T> &in, const forge::Image* image)
         // Map resource. Copy data to pixels. Unmap resource.
         size_t num_bytes;
         T* d_pixels = NULL;
-        cudaGraphicsMapResources(1, resources, cuda::getStream(cuda::getActiveDeviceId()));
+        cudaGraphicsMapResources(1, resources, cuda::getActiveStream());
         cudaGraphicsResourceGetMappedPointer((void **)&d_pixels, &num_bytes, resources[0]);
         cudaMemcpyAsync(d_pixels, d_X, num_bytes, cudaMemcpyDeviceToDevice,
-                        cuda::getStream(cuda::getActiveDeviceId()));
-        cudaGraphicsUnmapResources(1, resources, cuda::getStream(cuda::getActiveDeviceId()));
+                        cuda::getActiveStream());
+        cudaGraphicsUnmapResources(1, resources, cuda::getActiveStream());
 
         POST_LAUNCH_CHECK();
         CheckGL("After cuda resource copy");
diff --git a/src/backend/cuda/jit.cpp b/src/backend/cuda/jit.cpp
index 44fa9e1616..7d4111874a 100644
--- a/src/backend/cuda/jit.cpp
+++ b/src/backend/cuda/jit.cpp
@@ -655,7 +655,7 @@ void evalNodes(vector<Param<T> >&outputs, vector<Node *> nodes)
                             threads_y,
                             1,
                             0,
-                            getStream(getActiveDeviceId()),
+                            getActiveStream(),
                             &args.front(),
                             NULL));
 
diff --git a/src/backend/cuda/kernel/convolve.cu b/src/backend/cuda/kernel/convolve.cu
index 0a7f5425cf..d237fd8ca1 100644
--- a/src/backend/cuda/kernel/convolve.cu
+++ b/src/backend/cuda/kernel/convolve.cu
@@ -372,7 +372,7 @@ void convolve_1d(conv_kparam_t &p, Param<T> out, CParam<T> sig, CParam<aT> filt)
                                               filt.ptr+(f1Off+f2Off+f3Off),
                                               filterLen*sizeof(aT),
                                               0, cudaMemcpyDeviceToDevice,
-                                              cuda::getStream(cuda::getActiveDeviceId())));
+                                              cuda::getActiveStream()));
 
                 p.o[0] = (p.outHasNoOffset ? 0 : b1);
                 p.o[1] = (p.outHasNoOffset ? 0 : b2);
@@ -410,7 +410,7 @@ void convolve_2d(conv_kparam_t &p, Param<T> out, CParam<T> sig, CParam<aT> filt)
                                           filt.ptr+(f2Off+f3Off),
                                           filterLen*sizeof(aT),
                                           0, cudaMemcpyDeviceToDevice,
-                                          cuda::getStream(cuda::getActiveDeviceId())));
+                                          cuda::getActiveStream()));
 
             p.o[1] = (p.outHasNoOffset ? 0 : b2);
             p.o[2] = (p.outHasNoOffset ? 0 : b3);
@@ -438,7 +438,7 @@ void convolve_3d(conv_kparam_t &p, Param<T> out, CParam<T> sig, CParam<aT> filt)
                     filt.ptr+f3Off,
                     filterLen*sizeof(aT),
                     0, cudaMemcpyDeviceToDevice,
-                    cuda::getStream(cuda::getActiveDeviceId())));
+                    cuda::getActiveStream()));
 
         p.o[2] = (p.outHasNoOffset ? 0 : b3);
         p.s[2] = (p.inHasNoOffset ? 0 : b3);
diff --git a/src/backend/cuda/kernel/convolve_separable.cu b/src/backend/cuda/kernel/convolve_separable.cu
index 1b34e64043..65d7901a2c 100644
--- a/src/backend/cuda/kernel/convolve_separable.cu
+++ b/src/backend/cuda/kernel/convolve_separable.cu
@@ -133,7 +133,7 @@ void convolve2(Param<T> out, CParam<T> signal, CParam<accType> filter)
    // FIX ME: if the filter array is strided, direct copy of symbols
    // might cause issues
    CUDA_CHECK(cudaMemcpyToSymbolAsync(kernel::sFilter, filter.ptr, fLen*sizeof(accType), 0,
-               cudaMemcpyDeviceToDevice, cuda::getStream(cuda::getActiveDeviceId())));
+               cudaMemcpyDeviceToDevice, cuda::getActiveStream()));
 
     switch(fLen) {
         case  2: conv2Helper<T, accType, conv_dim, expand,  2>(blocks, threads, out, signal, blk_x, blk_y); break;
diff --git a/src/backend/cuda/kernel/fast.hpp b/src/backend/cuda/kernel/fast.hpp
index c50a3a1a3c..491ffe5f01 100644
--- a/src/backend/cuda/kernel/fast.hpp
+++ b/src/backend/cuda/kernel/fast.hpp
@@ -447,7 +447,7 @@ void fast(unsigned* out_feat,
     blocks.y = divup(in.dims[1], 64);
 
     unsigned *d_total = (unsigned *)(d_score + in.dims[0] * in.dims[1]);
-    CUDA_CHECK(cudaMemsetAsync(d_total, 0, sizeof(unsigned), cuda::getStream(cuda::getActiveDeviceId())));
+    CUDA_CHECK(cudaMemsetAsync(d_total, 0, sizeof(unsigned), cuda::getActiveStream()));
     unsigned *d_counts  = memAlloc<unsigned>(blocks.x * blocks.y);
     unsigned *d_offsets = memAlloc<unsigned>(blocks.x * blocks.y);
 
@@ -465,8 +465,8 @@ void fast(unsigned* out_feat,
     // Dimensions of output array
     unsigned total;
     CUDA_CHECK(cudaMemcpyAsync(&total, d_total, sizeof(unsigned), cudaMemcpyDeviceToHost,
-                cuda::getStream(cuda::getActiveDeviceId())));
-    CUDA_CHECK(cudaStreamSynchronize(cuda::getStream(cuda::getActiveDeviceId())));
+                cuda::getActiveStream()));
+    CUDA_CHECK(cudaStreamSynchronize(cuda::getActiveStream()));
     total = total < max_feat ? total : max_feat;
 
     if (total > 0) {
diff --git a/src/backend/cuda/kernel/harris.hpp b/src/backend/cuda/kernel/harris.hpp
index 11e6003158..cae2ae935e 100644
--- a/src/backend/cuda/kernel/harris.hpp
+++ b/src/backend/cuda/kernel/harris.hpp
@@ -216,7 +216,7 @@ void harris(unsigned* corners_out,
     int filter_elem = filter.strides[3] * filter.dims[3];
     filter.ptr = memAlloc<convAccT>(filter_elem);
     CUDA_CHECK(cudaMemcpyAsync(filter.ptr, h_filter, filter_elem * sizeof(convAccT),
-                cudaMemcpyHostToDevice, cuda::getStream(cuda::getActiveDeviceId())));
+                cudaMemcpyHostToDevice, cuda::getActiveStream()));
 
     delete[] h_filter;
 
@@ -277,7 +277,7 @@ void harris(unsigned* corners_out,
 
     unsigned* d_corners_found = memAlloc<unsigned>(1);
     CUDA_CHECK(cudaMemsetAsync(d_corners_found, 0, sizeof(unsigned),
-                cuda::getStream(cuda::getActiveDeviceId())));
+                cuda::getActiveStream()));
 
     float* d_x_corners = memAlloc<float>(corner_lim);
     float* d_y_corners = memAlloc<float>(corner_lim);
@@ -306,8 +306,8 @@ void harris(unsigned* corners_out,
 
     unsigned corners_found = 0;
     CUDA_CHECK(cudaMemcpyAsync(&corners_found, d_corners_found, sizeof(unsigned),
-                cudaMemcpyDeviceToHost, cuda::getStream(cuda::getActiveDeviceId())));
-    CUDA_CHECK(cudaStreamSynchronize(cuda::getStream(cuda::getActiveDeviceId())));
+                cudaMemcpyDeviceToHost, cuda::getActiveStream()));
+    CUDA_CHECK(cudaStreamSynchronize(cuda::getActiveStream()));
 
     memFree(d_responses);
     memFree(d_corners_found);
@@ -364,11 +364,11 @@ void harris(unsigned* corners_out,
         *y_out = memAlloc<float>(*corners_out);
         *resp_out = memAlloc<float>(*corners_out);
         CUDA_CHECK(cudaMemcpyAsync(*x_out, d_x_corners, *corners_out * sizeof(float),
-                    cudaMemcpyDeviceToDevice, cuda::getStream(cuda::getActiveDeviceId())));
+                    cudaMemcpyDeviceToDevice, cuda::getActiveStream()));
         CUDA_CHECK(cudaMemcpyAsync(*y_out, d_y_corners, *corners_out * sizeof(float),
-                    cudaMemcpyDeviceToDevice, cuda::getStream(cuda::getActiveDeviceId())));
+                    cudaMemcpyDeviceToDevice, cuda::getActiveStream()));
         CUDA_CHECK(cudaMemcpyAsync(*resp_out, d_resp_corners, *corners_out * sizeof(float),
-                    cudaMemcpyDeviceToDevice, cuda::getStream(cuda::getActiveDeviceId())));
+                    cudaMemcpyDeviceToDevice, cuda::getActiveStream()));
 
         memFree(d_x_corners);
         memFree(d_y_corners);
diff --git a/src/backend/cuda/kernel/homography.hpp b/src/backend/cuda/kernel/homography.hpp
index 23eb043c57..051fdb875e 100644
--- a/src/backend/cuda/kernel/homography.hpp
+++ b/src/backend/cuda/kernel/homography.hpp
@@ -612,27 +612,27 @@ int computeH(
             POST_LAUNCH_CHECK();
 
             CUDA_CHECK(cudaMemcpyAsync(&minMedian, finalMedian, sizeof(float),
-                        cudaMemcpyDeviceToHost, cuda::getStream(cuda::getActiveDeviceId())));
+                        cudaMemcpyDeviceToHost, cuda::getActiveStream()));
             CUDA_CHECK(cudaMemcpyAsync(&minIdx, finalIdx, sizeof(unsigned),
-                        cudaMemcpyDeviceToHost, cuda::getStream(cuda::getActiveDeviceId())));
+                        cudaMemcpyDeviceToHost, cuda::getActiveStream()));
 
             memFree(finalMedian);
             memFree(finalIdx);
         } else {
             CUDA_CHECK(cudaMemcpyAsync(&minMedian, median.ptr, sizeof(float),
-                        cudaMemcpyDeviceToHost, cuda::getStream(cuda::getActiveDeviceId())));
+                        cudaMemcpyDeviceToHost, cuda::getActiveStream()));
             CUDA_CHECK(cudaMemcpyAsync(&minIdx, idx.ptr, sizeof(unsigned),
-                        cudaMemcpyDeviceToHost, cuda::getStream(cuda::getActiveDeviceId())));
+                        cudaMemcpyDeviceToHost, cuda::getActiveStream()));
         }
 
         // Copy best homography to output
         CUDA_CHECK(cudaMemcpyAsync(bestH.ptr, H.ptr + minIdx * 9, 9*sizeof(T),
-                    cudaMemcpyDeviceToDevice, cuda::getStream(cuda::getActiveDeviceId())));
+                    cudaMemcpyDeviceToDevice, cuda::getActiveStream()));
 
         blocks = dim3(divup(nsamples, threads.x));
         // sync stream for the device to host copies to be visible for
         // the subsequent kernel launch
-        CUDA_CHECK(cudaStreamSynchronize(cuda::getStream(cuda::getActiveDeviceId())));
+        CUDA_CHECK(cudaStreamSynchronize(cuda::getActiveStream()));
 
         CUDA_LAUNCH((computeLMedSInliers<T>), blocks, threads,
                     inliers, bestH, x_src, y_src, x_dst, y_dst,
@@ -648,7 +648,7 @@ int computeH(
         kernel::reduce<unsigned, unsigned, af_add_t>(totalInliers, inliers, 0, false, 0.0);
 
         CUDA_CHECK(cudaMemcpyAsync(&inliersH, totalInliers.ptr, sizeof(unsigned),
-                    cudaMemcpyDeviceToHost, cuda::getStream(cuda::getActiveDeviceId())));
+                    cudaMemcpyDeviceToHost, cuda::getActiveStream()));
 
         memFree(totalInliers.ptr);
         memFree(median.ptr);
@@ -657,9 +657,9 @@ int computeH(
         inliersH = kernel::ireduce_all<unsigned, af_max_t>(&blockIdx, inliers);
         // Copies back index and number of inliers of best homography estimation
         CUDA_CHECK(cudaMemcpyAsync(&idxH, idx.ptr+blockIdx, sizeof(unsigned),
-                    cudaMemcpyDeviceToHost, cuda::getStream(cuda::getActiveDeviceId())));
+                    cudaMemcpyDeviceToHost, cuda::getActiveStream()));
         CUDA_CHECK(cudaMemcpyAsync(bestH.ptr, H.ptr + idxH * 9, 9*sizeof(T),
-                    cudaMemcpyDeviceToDevice, cuda::getStream(cuda::getActiveDeviceId())));
+                    cudaMemcpyDeviceToDevice, cuda::getActiveStream()));
 
     }
 
@@ -667,7 +667,7 @@ int computeH(
     memFree(idx.ptr);
     // sync stream for the device to host copies to be visible for
     // the subsequent kernel launch
-    CUDA_CHECK(cudaStreamSynchronize(cuda::getStream(cuda::getActiveDeviceId())));
+    CUDA_CHECK(cudaStreamSynchronize(cuda::getActiveStream()));
 
     return (int)inliersH;
 }
diff --git a/src/backend/cuda/kernel/ireduce.hpp b/src/backend/cuda/kernel/ireduce.hpp
index 12d4a8956f..e87360eb76 100644
--- a/src/backend/cuda/kernel/ireduce.hpp
+++ b/src/backend/cuda/kernel/ireduce.hpp
@@ -491,10 +491,10 @@ namespace kernel
             uint*   h_lptr_raw = h_lptr.get();
 
             CUDA_CHECK(cudaMemcpyAsync(h_ptr_raw, tmp.ptr, tmp_elements * sizeof(T),
-                       cudaMemcpyDeviceToHost, cuda::getStream(cuda::getActiveDeviceId())));
+                       cudaMemcpyDeviceToHost, cuda::getActiveStream()));
             CUDA_CHECK(cudaMemcpyAsync(h_lptr_raw, tlptr, tmp_elements * sizeof(uint),
-                       cudaMemcpyDeviceToHost, cuda::getStream(cuda::getActiveDeviceId())));
-            CUDA_CHECK(cudaStreamSynchronize(cuda::getStream(cuda::getActiveDeviceId())));
+                       cudaMemcpyDeviceToHost, cuda::getActiveStream()));
+            CUDA_CHECK(cudaStreamSynchronize(cuda::getActiveStream()));
             memFree(tmp.ptr);
             memFree(tlptr);
 
@@ -522,8 +522,8 @@ namespace kernel
             unique_ptr<T> h_ptr(new T[in_elements]);
             T* h_ptr_raw = h_ptr.get();
             CUDA_CHECK(cudaMemcpyAsync(h_ptr_raw, in.ptr, in_elements * sizeof(T),
-                       cudaMemcpyDeviceToHost, cuda::getStream(cuda::getActiveDeviceId())));
-            CUDA_CHECK(cudaStreamSynchronize(cuda::getStream(cuda::getActiveDeviceId())));
+                       cudaMemcpyDeviceToHost, cuda::getActiveStream()));
+            CUDA_CHECK(cudaStreamSynchronize(cuda::getActiveStream()));
 
             MinMaxOp<op, T> Op(h_ptr_raw[0], 0);
             for (int i = 1; i < in_elements; i++) {
diff --git a/src/backend/cuda/kernel/orb.hpp b/src/backend/cuda/kernel/orb.hpp
index d0cd2e7d3b..9b6c67bfd8 100644
--- a/src/backend/cuda/kernel/orb.hpp
+++ b/src/backend/cuda/kernel/orb.hpp
@@ -328,7 +328,7 @@ void orb(unsigned* out_feat,
     // In future implementations, the user will be capable of passing his
     // distribution instead of using the reference one
     //CUDA_CHECK(cudaMemcpyToSymbolAsync(d_ref_pat, h_ref_pat, 256 * 4 * sizeof(int), 0,
-    // cudaMemcpyHostToDevice, cuda::getStream(cuda::getActiveDeviceId())));
+    // cudaMemcpyHostToDevice, cuda::getActiveStream()));
 
     vector<float*> d_score_pyr(max_levels);
     vector<float*> d_ori_pyr(max_levels);
@@ -355,8 +355,8 @@ void orb(unsigned* out_feat,
         int gauss_elem = gauss_filter.strides[3] * gauss_filter.dims[3];
         gauss_filter.ptr = memAlloc<convAccT>(gauss_elem);
         CUDA_CHECK(cudaMemcpyAsync(gauss_filter.ptr, h_gauss.get(), gauss_elem * sizeof(convAccT),
-                    cudaMemcpyHostToDevice, cuda::getStream(cuda::getActiveDeviceId())));
-        CUDA_CHECK(cudaStreamSynchronize(cuda::getStream(cuda::getActiveDeviceId())));
+                    cudaMemcpyHostToDevice, cuda::getActiveStream()));
+        CUDA_CHECK(cudaStreamSynchronize(cuda::getActiveStream()));
     }
 
     for (int i = 0; i < (int)max_levels; i++) {
@@ -460,7 +460,7 @@ void orb(unsigned* out_feat,
 
         unsigned* d_desc_lvl = memAlloc<unsigned>(feat_pyr[i] * 8);
         CUDA_CHECK(cudaMemsetAsync(d_desc_lvl, 0, feat_pyr[i] * 8 * sizeof(unsigned),
-                    cuda::getStream(cuda::getActiveDeviceId())));
+                    cuda::getActiveStream()));
 
         // Compute ORB descriptors
         threads = dim3(THREADS_X, THREADS_Y);
@@ -508,17 +508,17 @@ void orb(unsigned* out_feat,
             offset += feat_pyr[i-1];
 
         CUDA_CHECK(cudaMemcpyAsync(*d_x+offset, d_x_pyr[i], feat_pyr[i] * sizeof(float),
-                    cudaMemcpyDeviceToDevice, cuda::getStream(cuda::getActiveDeviceId())));
+                    cudaMemcpyDeviceToDevice, cuda::getActiveStream()));
         CUDA_CHECK(cudaMemcpyAsync(*d_y+offset, d_y_pyr[i], feat_pyr[i] * sizeof(float),
-                    cudaMemcpyDeviceToDevice, cuda::getStream(cuda::getActiveDeviceId())));
+                    cudaMemcpyDeviceToDevice, cuda::getActiveStream()));
         CUDA_CHECK(cudaMemcpyAsync(*d_score+offset, d_score_pyr[i], feat_pyr[i] * sizeof(float),
-                    cudaMemcpyDeviceToDevice, cuda::getStream(cuda::getActiveDeviceId())));
+                    cudaMemcpyDeviceToDevice, cuda::getActiveStream()));
         CUDA_CHECK(cudaMemcpyAsync(*d_ori+offset, d_ori_pyr[i], feat_pyr[i] * sizeof(float),
-                    cudaMemcpyDeviceToDevice, cuda::getStream(cuda::getActiveDeviceId())));
+                    cudaMemcpyDeviceToDevice, cuda::getActiveStream()));
         CUDA_CHECK(cudaMemcpyAsync(*d_size+offset, d_size_pyr[i], feat_pyr[i] * sizeof(float),
-                    cudaMemcpyDeviceToDevice, cuda::getStream(cuda::getActiveDeviceId())));
+                    cudaMemcpyDeviceToDevice, cuda::getActiveStream()));
         CUDA_CHECK(cudaMemcpyAsync(*d_desc+(offset*8), d_desc_pyr[i], feat_pyr[i] * 8 * sizeof(unsigned),
-                    cudaMemcpyDeviceToDevice, cuda::getStream(cuda::getActiveDeviceId())));
+                    cudaMemcpyDeviceToDevice, cuda::getActiveStream()));
 
         memFree(d_x_pyr[i]);
         memFree(d_y_pyr[i]);
diff --git a/src/backend/cuda/kernel/reduce.hpp b/src/backend/cuda/kernel/reduce.hpp
index 3491fa69f1..30cbd05ff3 100644
--- a/src/backend/cuda/kernel/reduce.hpp
+++ b/src/backend/cuda/kernel/reduce.hpp
@@ -413,8 +413,8 @@ namespace kernel
             To* h_ptr_raw = h_ptr.get();
 
             CUDA_CHECK(cudaMemcpyAsync(h_ptr_raw, tmp.ptr, tmp_elements * sizeof(To),
-                       cudaMemcpyDeviceToHost, cuda::getStream(cuda::getActiveDeviceId())));
-            CUDA_CHECK(cudaStreamSynchronize(cuda::getStream(cuda::getActiveDeviceId())));
+                       cudaMemcpyDeviceToHost, cuda::getActiveStream()));
+            CUDA_CHECK(cudaStreamSynchronize(cuda::getActiveStream()));
             memFree(tmp.ptr);
 
             Binary<To, op> reduce;
@@ -430,8 +430,8 @@ namespace kernel
             unique_ptr<Ti> h_ptr(new Ti[in_elements]);
             Ti* h_ptr_raw = h_ptr.get();
             CUDA_CHECK(cudaMemcpyAsync(h_ptr_raw, in.ptr, in_elements * sizeof(Ti),
-                       cudaMemcpyDeviceToHost, cuda::getStream(cuda::getActiveDeviceId())));
-            CUDA_CHECK(cudaStreamSynchronize(cuda::getStream(cuda::getActiveDeviceId())));
+                       cudaMemcpyDeviceToHost, cuda::getActiveStream()));
+            CUDA_CHECK(cudaStreamSynchronize(cuda::getActiveStream()));
 
             Transform<Ti, To, op> transform;
             Binary<To, op> reduce;
diff --git a/src/backend/cuda/kernel/regions.hpp b/src/backend/cuda/kernel/regions.hpp
index 6c9abf3a02..d9ef4080a8 100644
--- a/src/backend/cuda/kernel/regions.hpp
+++ b/src/backend/cuda/kernel/regions.hpp
@@ -415,7 +415,7 @@ void regions(cuda::Param<T> out, cuda::CParam<char> in, cudaTextureObject_t tex)
         h_continue = 0;
         CUDA_CHECK(cudaMemcpyToSymbolAsync(continue_flag, &h_continue, sizeof(int),
                     0, cudaMemcpyHostToDevice,
-                    cuda::getStream(cuda::getActiveDeviceId())));
+                    cuda::getActiveStream()));
 
         CUDA_LAUNCH((update_equiv<T, 16, n_per_thread, full_conn>), blocks, threads, out, tex);
 
@@ -423,8 +423,8 @@ void regions(cuda::Param<T> out, cuda::CParam<char> in, cudaTextureObject_t tex)
 
         CUDA_CHECK(cudaMemcpyFromSymbolAsync(&h_continue, continue_flag, sizeof(int),
                     0, cudaMemcpyDeviceToHost,
-                    cuda::getStream(cuda::getActiveDeviceId())));
-        CUDA_CHECK(cudaStreamSynchronize(cuda::getStream(cuda::getActiveDeviceId())));
+                    cuda::getActiveStream()));
+        CUDA_CHECK(cudaStreamSynchronize(cuda::getActiveStream()));
     }
 
     // Now, perform the final relabeling.  This converts the equivalency
@@ -435,7 +435,7 @@ void regions(cuda::Param<T> out, cuda::CParam<char> in, cudaTextureObject_t tex)
     T* tmp = cuda::memAlloc<T>(size);
     CUDA_CHECK(cudaMemcpyAsync(tmp, out.ptr, size * sizeof(T),
                           cudaMemcpyDeviceToDevice,
-                          cuda::getStream(cuda::getActiveDeviceId())));
+                          cuda::getActiveStream()));
 
     // Wrap raw device ptr
     thrust::device_ptr<T> wrapped_tmp = thrust::device_pointer_cast(tmp);
diff --git a/src/backend/cuda/kernel/sift_nonfree.hpp b/src/backend/cuda/kernel/sift_nonfree.hpp
index 54b2a715db..5fa051b82d 100644
--- a/src/backend/cuda/kernel/sift_nonfree.hpp
+++ b/src/backend/cuda/kernel/sift_nonfree.hpp
@@ -190,8 +190,8 @@ Param<T> gauss_filter(float sigma)
     dim_t gauss_elem = gauss_filter.strides[3] * gauss_filter.dims[3];
     gauss_filter.ptr = memAlloc<T>(gauss_elem);
     CUDA_CHECK(cudaMemcpyAsync(gauss_filter.ptr, h_gauss, gauss_elem * sizeof(T),
-                cudaMemcpyHostToDevice, cuda::getStream(cuda::getActiveDeviceId())));
-    CUDA_CHECK(cudaStreamSynchronize(cuda::getStream(cuda::getActiveDeviceId())));
+                cudaMemcpyHostToDevice, cuda::getActiveStream()));
+    CUDA_CHECK(cudaStreamSynchronize(cuda::getActiveStream()));
 
     delete[] h_gauss;
 
@@ -1239,7 +1239,7 @@ std::vector< Param<T> > buildGaussPyr(
 
             CUDA_CHECK(cudaMemcpyAsync(gauss_pyr[o].ptr + offset, tmp_pyr[idx].ptr,
                         imel * sizeof(T), cudaMemcpyDeviceToDevice,
-                        cuda::getStream(cuda::getActiveDeviceId())));
+                        cuda::getActiveStream()));
         }
     }
 
@@ -1360,7 +1360,7 @@ void sift(unsigned* out_feat,
         const unsigned max_feat = ceil(imel * feature_ratio);
 
         CUDA_CHECK(cudaMemsetAsync(d_count, 0, sizeof(unsigned),
-                                   cuda::getStream(cuda::getActiveDeviceId())));
+                                   cuda::getActiveStream()));
 
         float* d_extrema_x = memAlloc<float>(max_feat);
         float* d_extrema_y = memAlloc<float>(max_feat);
@@ -1381,8 +1381,8 @@ void sift(unsigned* out_feat,
 
         unsigned extrema_feat = 0;
         CUDA_CHECK(cudaMemcpyAsync(&extrema_feat, d_count, sizeof(unsigned), cudaMemcpyDeviceToHost,
-                    cuda::getStream(cuda::getActiveDeviceId())));
-        CUDA_CHECK(cudaStreamSynchronize(cuda::getStream(cuda::getActiveDeviceId())));
+                    cuda::getActiveStream()));
+        CUDA_CHECK(cudaStreamSynchronize(cuda::getActiveStream()));
         extrema_feat = min(extrema_feat, max_feat);
 
         if (extrema_feat == 0) {
@@ -1394,7 +1394,7 @@ void sift(unsigned* out_feat,
         }
 
         CUDA_CHECK(cudaMemsetAsync(d_count, 0, sizeof(unsigned),
-                                   cuda::getStream(cuda::getActiveDeviceId())));
+                                   cuda::getActiveStream()));
 
         unsigned interp_feat = 0;
 
@@ -1420,12 +1420,12 @@ void sift(unsigned* out_feat,
         memFree(d_extrema_layer);
 
         CUDA_CHECK(cudaMemcpyAsync(&interp_feat, d_count, sizeof(unsigned), cudaMemcpyDeviceToHost,
-                    cuda::getStream(cuda::getActiveDeviceId())));
-        CUDA_CHECK(cudaStreamSynchronize(cuda::getStream(cuda::getActiveDeviceId())));
+                    cuda::getActiveStream()));
+        CUDA_CHECK(cudaStreamSynchronize(cuda::getActiveStream()));
         interp_feat = min(interp_feat, max_feat);
 
         CUDA_CHECK(cudaMemsetAsync(d_count, 0, sizeof(unsigned),
-                                   cuda::getStream(cuda::getActiveDeviceId())));
+                                   cuda::getActiveStream()));
 
         if (interp_feat == 0) {
             memFree(d_interp_x);
@@ -1482,10 +1482,10 @@ void sift(unsigned* out_feat,
 
         unsigned nodup_feat = 0;
         CUDA_CHECK(cudaMemcpyAsync(&nodup_feat, d_count, sizeof(unsigned), cudaMemcpyDeviceToHost,
-                    cuda::getStream(cuda::getActiveDeviceId())));
-        CUDA_CHECK(cudaStreamSynchronize(cuda::getStream(cuda::getActiveDeviceId())));
+                    cuda::getActiveStream()));
+        CUDA_CHECK(cudaStreamSynchronize(cuda::getActiveStream()));
         CUDA_CHECK(cudaMemsetAsync(d_count, 0, sizeof(unsigned),
-                                   cuda::getStream(cuda::getActiveDeviceId())));
+                                   cuda::getActiveStream()));
 
         const unsigned max_oriented_feat = nodup_feat * 3;
 
@@ -1516,8 +1516,8 @@ void sift(unsigned* out_feat,
 
         unsigned oriented_feat = 0;
         CUDA_CHECK(cudaMemcpyAsync(&oriented_feat, d_count, sizeof(unsigned), cudaMemcpyDeviceToHost,
-                    cuda::getStream(cuda::getActiveDeviceId())));
-        CUDA_CHECK(cudaStreamSynchronize(cuda::getStream(cuda::getActiveDeviceId())));
+                    cuda::getActiveStream()));
+        CUDA_CHECK(cudaStreamSynchronize(cuda::getActiveStream()));
         oriented_feat = min(oriented_feat, max_oriented_feat);
 
         if (oriented_feat == 0) {
@@ -1591,19 +1591,19 @@ void sift(unsigned* out_feat,
             continue;
 
         CUDA_CHECK(cudaMemcpyAsync(*d_x+offset, d_x_pyr[i], feat_pyr[i] * sizeof(float),
-                    cudaMemcpyDeviceToDevice, cuda::getStream(cuda::getActiveDeviceId())));
+                    cudaMemcpyDeviceToDevice, cuda::getActiveStream()));
         CUDA_CHECK(cudaMemcpyAsync(*d_y+offset, d_y_pyr[i], feat_pyr[i] * sizeof(float),
-                    cudaMemcpyDeviceToDevice, cuda::getStream(cuda::getActiveDeviceId())));
+                    cudaMemcpyDeviceToDevice, cuda::getActiveStream()));
         CUDA_CHECK(cudaMemcpyAsync(*d_score+offset, d_response_pyr[i], feat_pyr[i] * sizeof(float),
-                    cudaMemcpyDeviceToDevice, cuda::getStream(cuda::getActiveDeviceId())));
+                    cudaMemcpyDeviceToDevice, cuda::getActiveStream()));
         CUDA_CHECK(cudaMemcpyAsync(*d_ori+offset, d_ori_pyr[i], feat_pyr[i] * sizeof(float),
-                    cudaMemcpyDeviceToDevice, cuda::getStream(cuda::getActiveDeviceId())));
+                    cudaMemcpyDeviceToDevice, cuda::getActiveStream()));
         CUDA_CHECK(cudaMemcpyAsync(*d_size+offset, d_size_pyr[i], feat_pyr[i] * sizeof(float),
-                    cudaMemcpyDeviceToDevice, cuda::getStream(cuda::getActiveDeviceId())));
+                    cudaMemcpyDeviceToDevice, cuda::getActiveStream()));
 
         CUDA_CHECK(cudaMemcpyAsync(*d_desc+(offset*desc_len), d_desc_pyr[i],
                     feat_pyr[i] * desc_len * sizeof(float),
-                    cudaMemcpyDeviceToDevice, cuda::getStream(cuda::getActiveDeviceId())));
+                    cudaMemcpyDeviceToDevice, cuda::getActiveStream()));
 
         memFree(d_x_pyr[i]);
         memFree(d_y_pyr[i]);
diff --git a/src/backend/cuda/kernel/sort.hpp b/src/backend/cuda/kernel/sort.hpp
index 45dc023206..ac92ec8387 100644
--- a/src/backend/cuda/kernel/sort.hpp
+++ b/src/backend/cuda/kernel/sort.hpp
@@ -104,7 +104,7 @@ namespace cuda
             //val.modDims(inDims);
 
             // Not really necessary
-            // CUDA_CHECK(cudaStreamSynchronize(cuda::getStream(cuda::getActiveDeviceId())));
+            // CUDA_CHECK(cudaStreamSynchronize(cuda::getActiveStream()));
             memFree(key);
         }
 
diff --git a/src/backend/cuda/kernel/sort_by_key.hpp b/src/backend/cuda/kernel/sort_by_key.hpp
index a84e41c53d..8095da2d16 100644
--- a/src/backend/cuda/kernel/sort_by_key.hpp
+++ b/src/backend/cuda/kernel/sort_by_key.hpp
@@ -82,7 +82,7 @@ namespace cuda
             Tk *cKey = memAlloc<Tk>(elements);
             CUDA_CHECK(cudaMemcpyAsync(cKey, Key, elements * sizeof(Tk),
                                        cudaMemcpyDeviceToDevice,
-                                       getStream(cuda::getActiveDeviceId())));
+                                       getActiveStream()));
 
             Tv *Val = pVal.ptr;
             thrustSortByKey(Key, Val, elements, isAscending);
@@ -91,7 +91,7 @@ namespace cuda
             uint *cSeq = memAlloc<uint>(elements);
             CUDA_CHECK(cudaMemcpyAsync(cSeq, Seq, elements * sizeof(uint),
                                        cudaMemcpyDeviceToDevice,
-                                       getStream(cuda::getActiveDeviceId())));
+                                       getActiveStream()));
 
             // This always needs to be ascending
             thrustSortByKey(Seq, Val, elements, true);
diff --git a/src/backend/cuda/kernel/susan.hpp b/src/backend/cuda/kernel/susan.hpp
index 5c11e35367..c0ecc7f000 100644
--- a/src/backend/cuda/kernel/susan.hpp
+++ b/src/backend/cuda/kernel/susan.hpp
@@ -163,7 +163,7 @@ void nonMaximal(float* x_out, float* y_out, float* resp_out,
 
     unsigned* d_corners_found = memAlloc<unsigned>(1);
     CUDA_CHECK(cudaMemsetAsync(d_corners_found, 0, sizeof(unsigned),
-                cuda::getStream(cuda::getActiveDeviceId())));
+                cuda::getActiveStream()));
 
     CUDA_LAUNCH((nonMaxKernel<T>), blocks, threads,
             x_out, y_out, resp_out, d_corners_found, idim0, idim1, resp_in, edge, max_corners);
@@ -171,8 +171,8 @@ void nonMaximal(float* x_out, float* y_out, float* resp_out,
     POST_LAUNCH_CHECK();
 
     CUDA_CHECK(cudaMemcpyAsync(count, d_corners_found, sizeof(unsigned),
-                cudaMemcpyDeviceToHost, cuda::getStream(cuda::getActiveDeviceId())));
-    CUDA_CHECK(cudaStreamSynchronize(cuda::getStream(cuda::getActiveDeviceId())));
+                cudaMemcpyDeviceToHost, cuda::getActiveStream()));
+    CUDA_CHECK(cudaStreamSynchronize(cuda::getActiveStream()));
     memFree(d_corners_found);
 }
 
diff --git a/src/backend/cuda/kernel/transform.hpp b/src/backend/cuda/kernel/transform.hpp
index 735eaa5408..f3f551a3c6 100644
--- a/src/backend/cuda/kernel/transform.hpp
+++ b/src/backend/cuda/kernel/transform.hpp
@@ -199,7 +199,7 @@ namespace cuda
             CUDA_CHECK(cudaMemcpyToSymbolAsync(c_tmat, tf.ptr,
                                                nTfs2 * nTfs3 * tf_len * sizeof(float),
                                                0, cudaMemcpyDeviceToDevice,
-                                               cuda::getStream(cuda::getActiveDeviceId())));
+                                               cuda::getActiveStream()));
 
             dim3 threads(TX, TY, 1);
             dim3 blocks(divup(out.dims[0], threads.x), divup(out.dims[1], threads.y));
diff --git a/src/backend/cuda/kernel/where.hpp b/src/backend/cuda/kernel/where.hpp
index 8df9b506de..c6c5894367 100644
--- a/src/backend/cuda/kernel/where.hpp
+++ b/src/backend/cuda/kernel/where.hpp
@@ -118,8 +118,8 @@ namespace kernel
         uint total;
         CUDA_CHECK(cudaMemcpyAsync(&total, rtmp.ptr + rtmp_elements - 1,
                               sizeof(uint), cudaMemcpyDeviceToHost,
-                              cuda::getStream(cuda::getActiveDeviceId())));
-        CUDA_CHECK(cudaStreamSynchronize(cuda::getStream(cuda::getActiveDeviceId())));
+                              cuda::getActiveStream()));
+        CUDA_CHECK(cudaStreamSynchronize(cuda::getActiveStream()));
 
         out.ptr = memAlloc<uint>(total);
 
diff --git a/src/backend/cuda/morph3d_impl.hpp b/src/backend/cuda/morph3d_impl.hpp
index d5a19f7128..e6233c989e 100644
--- a/src/backend/cuda/morph3d_impl.hpp
+++ b/src/backend/cuda/morph3d_impl.hpp
@@ -34,7 +34,7 @@ Array<T> morph3d(const Array<T> &in, const Array<T> &mask)
     CUDA_CHECK(cudaMemcpyToSymbolAsync(kernel::cFilter, mask.get(),
                                   mdims[0] * mdims[1] *mdims[2] * sizeof(T),
                                   0, cudaMemcpyDeviceToDevice,
-                                  cuda::getStream(cuda::getActiveDeviceId())));
+                                  cuda::getActiveStream()));
 
     if (isDilation)
         kernel::morph3d<T, true >(out, in, mdims[0]);
diff --git a/src/backend/cuda/morph_impl.hpp b/src/backend/cuda/morph_impl.hpp
index 16437985bd..994e792a4b 100644
--- a/src/backend/cuda/morph_impl.hpp
+++ b/src/backend/cuda/morph_impl.hpp
@@ -33,7 +33,7 @@ Array<T>  morph(const Array<T> &in, const Array<T> &mask)
     CUDA_CHECK(cudaMemcpyToSymbolAsync(kernel::cFilter, mask.get(),
                                   mdims[0] * mdims[1] * sizeof(T),
                                   0, cudaMemcpyDeviceToDevice,
-                                  cuda::getStream(cuda::getActiveDeviceId())));
+                                  cuda::getActiveStream()));
 
     if (isDilation)
         kernel::morph<T, true >(out, in, mdims[0]);
diff --git a/src/backend/cuda/platform.cpp b/src/backend/cuda/platform.cpp
index 3679d66ed6..ca00ccf06c 100644
--- a/src/backend/cuda/platform.cpp
+++ b/src/backend/cuda/platform.cpp
@@ -327,6 +327,11 @@ cudaStream_t getStream(int device)
     return str;
 }
 
+cudaStream_t getActiveStream()
+{
+    return getStream(getActiveDeviceId());
+}
+
 size_t getDeviceMemorySize(int device)
 {
     return getDeviceProp(device).totalGlobalMem;
@@ -601,7 +606,7 @@ void sync(int device)
 {
     int currDevice = getActiveDeviceId();
     setDevice(device);
-    CUDA_CHECK(cudaStreamSynchronize(getStream(getActiveDeviceId())));
+    CUDA_CHECK(cudaStreamSynchronize(getActiveStream()));
     setDevice(currDevice);
 }
 
diff --git a/src/backend/cuda/platform.hpp b/src/backend/cuda/platform.hpp
index d665e9539b..5a025621d3 100644
--- a/src/backend/cuda/platform.hpp
+++ b/src/backend/cuda/platform.hpp
@@ -49,6 +49,8 @@ int getDeviceNativeId(int device);
 
 cudaStream_t getStream(int device);
 
+cudaStream_t getActiveStream();
+
 size_t getDeviceMemorySize(int device);
 
 size_t getHostMemorySize();
diff --git a/src/backend/cuda/plot.cpp b/src/backend/cuda/plot.cpp
index f2ca1f19cf..3c67e7d36b 100644
--- a/src/backend/cuda/plot.cpp
+++ b/src/backend/cuda/plot.cpp
@@ -36,11 +36,11 @@ void copy_plot(const Array<T> &P, forge::Plot* plot)
         // Map resource. Copy data to VBO. Unmap resource.
         size_t num_bytes = plot->verticesSize();
         T* d_vbo = NULL;
-        cudaGraphicsMapResources(1, resources, cuda::getStream(cuda::getActiveDeviceId()));
+        cudaGraphicsMapResources(1, resources, cuda::getActiveStream());
         cudaGraphicsResourceGetMappedPointer((void **)&d_vbo, &num_bytes, resources[0]);
         cudaMemcpyAsync(d_vbo, d_P, num_bytes, cudaMemcpyDeviceToDevice,
-                cuda::getStream(cuda::getActiveDeviceId()));
-        cudaGraphicsUnmapResources(1, resources, cuda::getStream(cuda::getActiveDeviceId()));
+                cuda::getActiveStream());
+        cudaGraphicsUnmapResources(1, resources, cuda::getActiveStream());
 
         CheckGL("After cuda resource copy");
 
diff --git a/src/backend/cuda/sparse.cu b/src/backend/cuda/sparse.cu
index 6ce7c9226b..49eb442d99 100644
--- a/src/backend/cuda/sparse.cu
+++ b/src/backend/cuda/sparse.cu
@@ -373,7 +373,7 @@ SparseArray<T> sparseConvertStorageToStorage(const SparseArray<T> &in)
         CUDA_CHECK(cudaMemcpyAsync(converted.getColIdx().get(), in.getColIdx().get(),
                                    in.getColIdx().elements() * sizeof(int),
                                    cudaMemcpyDeviceToDevice,
-                                   cuda::getStream(cuda::getActiveDeviceId())));
+                                   cuda::getActiveStream()));
 
         // cusparse function to expand compressed row into coordinate
         CUSPARSE_CHECK(cusparseXcsr2coo(
@@ -448,11 +448,11 @@ SparseArray<T> sparseConvertStorageToStorage(const SparseArray<T> &in)
         CUDA_CHECK(cudaMemcpyAsync(converted.getValues().get(), cooT.getValues().get(),
                                    cooT.getValues().elements() * sizeof(T),
                                    cudaMemcpyDeviceToDevice,
-                                   cuda::getStream(cuda::getActiveDeviceId())));
+                                   cuda::getActiveStream()));
         CUDA_CHECK(cudaMemcpyAsync(converted.getColIdx().get(), cooT.getColIdx().get(),
                                    cooT.getColIdx().elements() * sizeof(int),
                                    cudaMemcpyDeviceToDevice,
-                                   cuda::getStream(cuda::getActiveDeviceId())));
+                                   cuda::getActiveStream()));
 
         // cusparse function to compress row from coordinate
         CUSPARSE_CHECK(cusparseXcoo2csr(
diff --git a/src/backend/cuda/surface.cpp b/src/backend/cuda/surface.cpp
index 0752a950eb..95ff72645f 100644
--- a/src/backend/cuda/surface.cpp
+++ b/src/backend/cuda/surface.cpp
@@ -36,11 +36,11 @@ void copy_surface(const Array<T> &P, forge::Surface* surface)
         // Map resource. Copy data to VBO. Unmap resource.
         size_t num_bytes = surface->verticesSize();
         T* d_vbo = NULL;
-        cudaGraphicsMapResources(1, resources, cuda::getStream(cuda::getActiveDeviceId()));
+        cudaGraphicsMapResources(1, resources, cuda::getActiveStream());
         cudaGraphicsResourceGetMappedPointer((void **)&d_vbo, &num_bytes, resources[0]);
         cudaMemcpyAsync(d_vbo, d_P, num_bytes, cudaMemcpyDeviceToDevice,
-                cuda::getStream(cuda::getActiveDeviceId()));
-        cudaGraphicsUnmapResources(1, resources, cuda::getStream(cuda::getActiveDeviceId()));
+                cuda::getActiveStream());
+        cudaGraphicsUnmapResources(1, resources, cuda::getActiveStream());
 
         CheckGL("After cuda resource copy");
 
diff --git a/src/backend/cuda/vector_field.cpp b/src/backend/cuda/vector_field.cpp
index 36021aa1c3..d6eb919422 100644
--- a/src/backend/cuda/vector_field.cpp
+++ b/src/backend/cuda/vector_field.cpp
@@ -32,7 +32,7 @@ void copy_vector_field(const Array<T> &points, const Array<T> &directions,
 
         // Map resource. Copy data to VBO. Unmap resource.
         // Map all resources at once.
-        cudaGraphicsMapResources(2, resources, cuda::getStream(cuda::getActiveDeviceId()));
+        cudaGraphicsMapResources(2, resources, cuda::getActiveStream());
 
         // Points
         {
@@ -41,7 +41,7 @@ void copy_vector_field(const Array<T> &points, const Array<T> &directions,
             T* d_vbo = NULL;
             cudaGraphicsResourceGetMappedPointer((void **)&d_vbo, &num_bytes, resources[0]);
             cudaMemcpyAsync(d_vbo, ptr, num_bytes, cudaMemcpyDeviceToDevice,
-                            cuda::getStream(cuda::getActiveDeviceId()));
+                            cuda::getActiveStream());
         }
         // Directions
         {
@@ -50,9 +50,9 @@ void copy_vector_field(const Array<T> &points, const Array<T> &directions,
             T* d_vbo = NULL;
             cudaGraphicsResourceGetMappedPointer((void **)&d_vbo, &num_bytes, resources[1]);
             cudaMemcpyAsync(d_vbo, ptr, num_bytes, cudaMemcpyDeviceToDevice,
-                            cuda::getStream(cuda::getActiveDeviceId()));
+                            cuda::getActiveStream());
         }
-        cudaGraphicsUnmapResources(2, resources, cuda::getStream(cuda::getActiveDeviceId()));
+        cudaGraphicsUnmapResources(2, resources, cuda::getActiveStream());
 
         CheckGL("After cuda resource copy");
 

From b2e54ff67c4687bc3f4368bf670ff1254fbe8808 Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Tue, 17 Jan 2017 17:57:13 +0530
Subject: [PATCH 26/82] Change memory manager classes to use CRTP

---
 src/backend/MemoryManager.cpp        | 331 ----------------------
 src/backend/MemoryManager.hpp        | 123 --------
 src/backend/common/MemoryManager.hpp | 402 +++++++++++++++++++++++++++
 src/backend/cpu/Array.cpp            |   1 -
 src/backend/cpu/memory.cpp           | 135 ++++-----
 src/backend/cpu/memory.hpp           |  56 ++--
 src/backend/cpu/platform.cpp         |  11 +
 src/backend/cpu/platform.hpp         |  11 +-
 src/backend/cuda/Array.cpp           |   1 -
 src/backend/cuda/memory.cpp          | 130 +++++++--
 src/backend/cuda/memory.hpp          |  71 +++--
 src/backend/cuda/memoryManager.cpp   |  86 ------
 src/backend/cuda/memoryManager.hpp   |  63 -----
 src/backend/cuda/platform.cpp        |  13 +-
 src/backend/cuda/platform.hpp        |  12 +-
 src/backend/opencl/Array.cpp         |   1 -
 src/backend/opencl/memory.cpp        | 146 ++++++++--
 src/backend/opencl/memory.hpp        |  76 +++--
 src/backend/opencl/memoryManager.cpp |  86 ------
 src/backend/opencl/memoryManager.hpp |  68 -----
 src/backend/opencl/platform.cpp      |  13 +-
 src/backend/opencl/platform.hpp      |  13 +-
 22 files changed, 868 insertions(+), 981 deletions(-)
 delete mode 100644 src/backend/MemoryManager.cpp
 delete mode 100644 src/backend/MemoryManager.hpp
 create mode 100644 src/backend/common/MemoryManager.hpp
 delete mode 100644 src/backend/cuda/memoryManager.cpp
 delete mode 100644 src/backend/cuda/memoryManager.hpp
 delete mode 100644 src/backend/opencl/memoryManager.cpp
 delete mode 100644 src/backend/opencl/memoryManager.hpp

diff --git a/src/backend/MemoryManager.cpp b/src/backend/MemoryManager.cpp
deleted file mode 100644
index 3b0e81ddc5..0000000000
--- a/src/backend/MemoryManager.cpp
+++ /dev/null
@@ -1,331 +0,0 @@
-/*******************************************************
- * Copyright (c) 2014, ArrayFire
- * All rights reserved.
- *
- * This file is distributed under 3-clause BSD license.
- * The complete license agreement can be obtained at:
- * http://arrayfire.com/licenses/BSD-3-Clause
- ********************************************************/
-
-#include <iostream>
-#include <iomanip>
-#include <string>
-#include <algorithm>
-#include "MemoryManager.hpp"
-#include "dispatch.hpp"
-#include "err_common.hpp"
-#include "util.hpp"
-
-namespace common
-{
-
-MemoryManager::MemoryManager(int num_devices, unsigned MAX_BUFFERS, bool debug):
-    mem_step_size(1024),
-    max_buffers(MAX_BUFFERS),
-    memory(num_devices),
-    debug_mode(debug)
-{
-    lock_guard_t lock(this->memory_mutex);
-
-    for (int n = 0; n < num_devices; n++) {
-        // Calling getMaxMemorySize() here calls the virtual function that returns 0
-        // Call it from outside the constructor.
-        memory[n].max_bytes     = ONE_GB;
-        memory[n].total_bytes   = 0;
-        memory[n].total_buffers = 0;
-        memory[n].lock_bytes    = 0;
-        memory[n].lock_buffers  = 0;
-    }
-
-    // Check for environment variables
-
-    std::string env_var;
-
-    // Debug mode
-    env_var = getEnvVar("AF_MEM_DEBUG");
-    if (!env_var.empty()) {
-        this->debug_mode = env_var[0] != '0';
-    }
-    if (this->debug_mode) mem_step_size = 1;
-
-    // Max Buffer count
-    env_var = getEnvVar("AF_MAX_BUFFERS");
-    if (!env_var.empty()) {
-        this->max_buffers = std::max(1, std::stoi(env_var));
-    }
-}
-
-void MemoryManager::setMaxMemorySize()
-{
-    for (unsigned n = 0; n < memory.size(); n++) {
-        // Calls garbage collection when:
-        // total_bytes > memsize * 0.75 when memsize <  4GB
-        // total_bytes > memsize - 1 GB when memsize >= 4GB
-        // If memsize returned 0, then use 1GB
-        size_t memsize = this->getMaxMemorySize(n);
-        memory[n].max_bytes = memsize == 0 ? ONE_GB : std::max(memsize * 0.75, (double)(memsize - ONE_GB));
-    }
-}
-
-void MemoryManager::garbageCollect()
-{
-    if (this->debug_mode) return;
-
-    lock_guard_t lock(this->memory_mutex);
-    memory_info& current = this->getCurrentMemoryInfo();
-
-    // Return if all buffers are locked
-    if (current.total_buffers == current.lock_buffers) return;
-
-    for (auto &kv : current.free_map) {
-        size_t num_ptrs = kv.second.size();
-        //Free memory by popping the last element
-        for (int n = num_ptrs-1; n >= 0; n--) {
-            this->nativeFree(kv.second[n]);
-            current.total_bytes -= kv.first;
-            current.total_buffers--;
-            kv.second.pop_back();
-        }
-    }
-    current.free_map.clear();
-}
-
-void MemoryManager::unlock(void *ptr, bool user_unlock)
-{
-    // Shortcut for empty arrays
-    if (!ptr) return;
-
-    lock_guard_t lock(this->memory_mutex);
-    memory_info& current = this->getCurrentMemoryInfo();
-
-    locked_iter iter = current.locked_map.find((void *)ptr);
-
-    // Pointer not found in locked map
-    if (iter == current.locked_map.end()) {
-        // Probably came from user, just free it
-        this->nativeFree(ptr);
-        return;
-    }
-
-    if (user_unlock) {
-        (iter->second).user_lock = false;
-    } else {
-        (iter->second).manager_lock = false;
-    }
-
-    // Return early if either one is locked
-    if ((iter->second).user_lock || (iter->second).manager_lock) return;
-
-    size_t bytes = iter->second.bytes;
-    current.lock_bytes -= iter->second.bytes;
-    current.lock_buffers--;
-
-    current.locked_map.erase(iter);
-
-    if (this->debug_mode) {
-        // Just free memory in debug mode
-        if ((iter->second).bytes > 0) {
-            this->nativeFree(iter->first);
-            current.total_buffers--;
-            current.total_bytes -= iter->second.bytes;
-        }
-    } else {
-        // In regular mode, move buffer to free map
-        free_iter fiter = current.free_map.find(bytes);
-        if (fiter != current.free_map.end()) {
-            // If found, push back
-            fiter->second.push_back(ptr);
-        } else {
-            // If not found, create new vector for this size
-            std::vector<void *> ptrs;
-            ptrs.push_back(ptr);
-            current.free_map[bytes] = ptrs;
-        }
-    }
-}
-
-void *MemoryManager::alloc(const size_t bytes, bool user_lock)
-{
-    lock_guard_t lock(this->memory_mutex);
-
-    void *ptr = NULL;
-    size_t alloc_bytes = this->debug_mode ? bytes : (divup(bytes, mem_step_size) * mem_step_size);
-
-    if (bytes > 0) {
-        memory_info& current = this->getCurrentMemoryInfo();
-
-        // There is no memory cache in debug mode
-        if (!this->debug_mode) {
-
-            // FIXME: Add better checks for garbage collection
-            // Perhaps look at total memory available as a metric
-            if (this->checkMemoryLimit()) {
-                this->garbageCollect();
-            }
-
-            free_iter iter = current.free_map.find(alloc_bytes);
-
-            if (iter != current.free_map.end() && !iter->second.empty()) {
-                ptr = iter->second.back();
-                iter->second.pop_back();
-            }
-
-        }
-
-        // Only comes here if buffer size not found or in debug mode
-        if (ptr == NULL) {
-            // Perform garbage collection if memory can not be allocated
-            try {
-                ptr = this->nativeAlloc(alloc_bytes);
-            } catch (AfError &ex) {
-                // If out of memory, run garbage collect and try again
-                if (ex.getError() != AF_ERR_NO_MEM) throw;
-                this->garbageCollect();
-                ptr = this->nativeAlloc(alloc_bytes);
-            }
-            // Increment these two only when it succeeds to come here.
-            current.total_bytes += alloc_bytes;
-            current.total_buffers += 1;
-        }
-
-
-        locked_info info = {!user_lock, user_lock, alloc_bytes};
-        current.locked_map[ptr] = info;
-        current.lock_bytes += alloc_bytes;
-        current.lock_buffers++;
-    }
-    return ptr;
-}
-
-void MemoryManager::userLock(const void *ptr)
-{
-    memory_info& current = this->getCurrentMemoryInfo();
-
-    lock_guard_t lock(this->memory_mutex);
-
-    locked_iter iter = current.locked_map.find(const_cast<void *>(ptr));
-
-    if (iter != current.locked_map.end()) {
-        iter->second.user_lock = true;
-    } else {
-        locked_info info = {false,
-                            true,
-                            100}; //This number is not relevant
-
-        current.locked_map[(void *)ptr] = info;
-    }
-}
-
-void MemoryManager::userUnlock(const void *ptr)
-{
-    this->unlock(const_cast<void *>(ptr), true);
-}
-
-bool MemoryManager::isUserLocked(const void *ptr)
-{
-    memory_info& current = this->getCurrentMemoryInfo();
-    lock_guard_t lock(this->memory_mutex);
-    locked_iter iter = current.locked_map.find(const_cast<void *>(ptr));
-    if (iter != current.locked_map.end()) {
-        return iter->second.user_lock;
-    } else {
-        return false;
-    }
-}
-
-size_t MemoryManager::getMemStepSize()
-{
-    lock_guard_t lock(this->memory_mutex);
-    return this->mem_step_size;
-}
-
-void MemoryManager::setMemStepSize(size_t new_step_size)
-{
-    lock_guard_t lock(this->memory_mutex);
-    this->mem_step_size = new_step_size;
-}
-
-size_t MemoryManager::getMaxBytes()
-{
-    lock_guard_t lock(this->memory_mutex);
-    return this->getCurrentMemoryInfo().max_bytes;
-}
-
-void MemoryManager::printInfo(const char *msg, const int device)
-{
-    lock_guard_t lock(this->memory_mutex);
-    const memory_info& current = this->getCurrentMemoryInfo();
-
-    std::cout << msg << std::endl;
-
-    static const std::string head("|     POINTER      |    SIZE    |  AF LOCK  | USER LOCK |");
-    static const std::string line(head.size(), '-');
-    std::cout << line << std::endl << head << std::endl << line << std::endl;
-
-    for(auto& kv : current.locked_map) {
-        std::string status_mngr("Yes");
-        std::string status_user("Unknown");
-        if(kv.second.user_lock)     status_user = "Yes";
-        else                        status_user = " No";
-
-        std::string unit = "KB";
-        double size = (double)(kv.second.bytes) / 1024;
-        if(size >= 1024) {
-            size = size / 1024;
-            unit = "MB";
-        }
-
-        std::cout << "|  " << std::right << std::setw(14) << kv.first << " "
-                  << " | " << std::setw(7) << std::setprecision(4) << size << " " << unit
-                  << " | " << std::setw(9) << status_mngr
-                  << " | " << std::setw(9) << status_user
-                  << " |"  << std::endl;
-    }
-
-    for(auto &kv : current.free_map) {
-
-        std::string status_mngr("No");
-        std::string status_user("No");
-
-        std::string unit = "KB";
-        double size = (double)(kv.first) / 1024;
-        if(size >= 1024) {
-            size = size / 1024;
-            unit = "MB";
-        }
-
-        for (auto &ptr : kv.second) {
-            std::cout << "|  " << std::right << std::setw(14) << ptr << " "
-                      << " | " << std::setw(7) << std::setprecision(4) << size << " " << unit
-                      << " | " << std::setw(9) << status_mngr
-                      << " | " << std::setw(9) << status_user
-                      << " |"  << std::endl;
-        }
-    }
-
-    std::cout << line << std::endl;
-}
-
-void MemoryManager::bufferInfo(size_t *alloc_bytes, size_t *alloc_buffers,
-                               size_t *lock_bytes,  size_t *lock_buffers)
-{
-    lock_guard_t lock(this->memory_mutex);
-    const memory_info& current = this->getCurrentMemoryInfo();
-    if (alloc_bytes   ) *alloc_bytes   = current.total_bytes;
-    if (alloc_buffers ) *alloc_buffers = current.total_buffers;
-    if (lock_bytes    ) *lock_bytes    = current.lock_bytes;
-    if (lock_buffers  ) *lock_buffers  = current.lock_buffers;
-}
-
-unsigned MemoryManager::getMaxBuffers()
-{
-    return this->max_buffers;
-}
-
-bool MemoryManager::checkMemoryLimit()
-{
-    const memory_info& current = this->getCurrentMemoryInfo();
-    return current.lock_bytes >= current.max_bytes || current.total_buffers >= this->max_buffers;
-}
-
-}
diff --git a/src/backend/MemoryManager.hpp b/src/backend/MemoryManager.hpp
deleted file mode 100644
index 39ff8e1281..0000000000
--- a/src/backend/MemoryManager.hpp
+++ /dev/null
@@ -1,123 +0,0 @@
-/*******************************************************
- * Copyright (c) 2014, ArrayFire
- * All rights reserved.
- *
- * This file is distributed under 3-clause BSD license.
- * The complete license agreement can be obtained at:
- * http://arrayfire.com/licenses/BSD-3-Clause
- ********************************************************/
-
-#pragma once
-
-#include <vector>
-#include <mutex>
-#include <unordered_map>
-
-namespace common
-{
-
-typedef std::recursive_mutex mutex_t;
-typedef std::lock_guard<mutex_t> lock_guard_t;
-
-const unsigned MAX_BUFFERS   = 1000;
-const size_t ONE_GB = 1 << 30;
-
-class MemoryManager
-{
-    typedef struct
-    {
-        bool manager_lock;
-        bool user_lock;
-        size_t bytes;
-    } locked_info;
-
-    typedef std::unordered_map<void *, locked_info> locked_t;
-    typedef locked_t::iterator locked_iter;
-
-    typedef std::unordered_map<size_t, std::vector<void *> >free_t;
-    typedef free_t::iterator free_iter;
-
-    typedef struct
-    {
-        locked_t locked_map;
-        free_t   free_map;
-
-        size_t lock_bytes;
-        size_t lock_buffers;
-        size_t total_bytes;
-        size_t total_buffers;
-        size_t max_bytes;
-    } memory_info;
-
-    size_t mem_step_size;
-    unsigned max_buffers;
-    std::vector<memory_info> memory;
-    bool debug_mode;
-
-    memory_info& getCurrentMemoryInfo()
-    {
-        return memory[this->getActiveDeviceId()];
-    }
-
-    virtual int getActiveDeviceId()
-    {
-        return 0;
-    }
-
-    virtual size_t getMaxMemorySize(int id)
-    {
-        return 0;
-    }
-
-public:
-    MemoryManager(int num_devices, unsigned MAX_BUFFERS, bool debug);
-
-    void setMaxMemorySize();
-
-    void *alloc(const size_t bytes, bool user_lock);
-
-    void unlock(void *ptr, bool user_unlock);
-
-    void garbageCollect();
-
-    void printInfo(const char *msg, const int device);
-
-    void bufferInfo(size_t *alloc_bytes, size_t *alloc_buffers,
-                    size_t *lock_bytes,  size_t *lock_buffers);
-
-    void userLock(const void *ptr);
-
-    void userUnlock(const void *ptr);
-
-    bool isUserLocked(const void *ptr);
-
-    size_t getMemStepSize();
-
-    size_t getMaxBytes();
-
-    unsigned getMaxBuffers();
-
-    void setMemStepSize(size_t new_step_size);
-
-    virtual void *nativeAlloc(const size_t bytes)
-    {
-        return malloc(bytes);
-    }
-
-    virtual void nativeFree(void *ptr)
-    {
-        free((void *)ptr);
-    }
-
-    virtual ~MemoryManager()
-    {
-    }
-
-    bool checkMemoryLimit();
-
-protected:
-    mutex_t memory_mutex;
-
-};
-
-}
diff --git a/src/backend/common/MemoryManager.hpp b/src/backend/common/MemoryManager.hpp
new file mode 100644
index 0000000000..892df3f50c
--- /dev/null
+++ b/src/backend/common/MemoryManager.hpp
@@ -0,0 +1,402 @@
+/*******************************************************
+ * Copyright (c) 2014, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <vector>
+#include <mutex>
+#include <unordered_map>
+#include <iostream>
+#include <iomanip>
+#include <err_common.hpp>
+#include <dispatch.hpp>
+#include <util.hpp>
+
+namespace common
+{
+typedef std::recursive_mutex mutex_t;
+typedef std::lock_guard<mutex_t> lock_guard_t;
+
+const unsigned MAX_BUFFERS   = 1000;
+const size_t ONE_GB = 1 << 30;
+
+template<typename T>
+class MemoryManager
+{
+    typedef struct
+    {
+        bool manager_lock;
+        bool user_lock;
+        size_t bytes;
+    } locked_info;
+
+    using locked_t = typename std::unordered_map<void *, locked_info>;
+    using locked_iter = typename locked_t::iterator;
+
+    typedef std::unordered_map<size_t, std::vector<void *> >free_t;
+    typedef free_t::iterator free_iter;
+
+    typedef struct
+    {
+        locked_t locked_map;
+        free_t   free_map;
+
+        size_t lock_bytes;
+        size_t lock_buffers;
+        size_t total_bytes;
+        size_t total_buffers;
+        size_t max_bytes;
+    } memory_info;
+
+    size_t mem_step_size;
+    unsigned max_buffers;
+    std::vector<memory_info> memory;
+    bool debug_mode;
+
+    memory_info& getCurrentMemoryInfo()
+    {
+        return memory[this->getActiveDeviceId()];
+    }
+
+    inline int getActiveDeviceId()
+    {
+        return static_cast<T*>(this)->getActiveDeviceId();
+    }
+
+    inline size_t getMaxMemorySize(int id)
+    {
+        return static_cast<T*>(this)->getMaxMemorySize(id);
+    }
+
+    public:
+    MemoryManager(int num_devices, unsigned MAX_BUFFERS, bool debug)
+        : mem_step_size(1024), max_buffers(MAX_BUFFERS), memory(num_devices), debug_mode(debug)
+    {
+        lock_guard_t lock(this->memory_mutex);
+
+        for (int n = 0; n < num_devices; n++) {
+            // Calling getMaxMemorySize() here calls the virtual function that returns 0
+            // Call it from outside the constructor.
+            memory[n].max_bytes     = ONE_GB;
+            memory[n].total_bytes   = 0;
+            memory[n].total_buffers = 0;
+            memory[n].lock_bytes    = 0;
+            memory[n].lock_buffers  = 0;
+        }
+
+        // Check for environment variables
+
+        std::string env_var;
+
+        // Debug mode
+        env_var = getEnvVar("AF_MEM_DEBUG");
+        if (!env_var.empty()) {
+            this->debug_mode = env_var[0] != '0';
+        }
+        if (this->debug_mode) mem_step_size = 1;
+
+        // Max Buffer count
+        env_var = getEnvVar("AF_MAX_BUFFERS");
+        if (!env_var.empty()) {
+            this->max_buffers = std::max(1, std::stoi(env_var));
+        }
+    }
+
+    void setMaxMemorySize()
+    {
+        for (unsigned n = 0; n < memory.size(); n++) {
+            // Calls garbage collection when:
+            // total_bytes > memsize * 0.75 when memsize <  4GB
+            // total_bytes > memsize - 1 GB when memsize >= 4GB
+            // If memsize returned 0, then use 1GB
+            size_t memsize = this->getMaxMemorySize(n);
+            memory[n].max_bytes = memsize == 0 ? ONE_GB :
+                std::max(memsize * 0.75, (double)(memsize - ONE_GB));
+        }
+    }
+
+    void *alloc(const size_t bytes, bool user_lock)
+    {
+        lock_guard_t lock(this->memory_mutex);
+
+        void *ptr = NULL;
+        size_t alloc_bytes = this->debug_mode ? bytes : (divup(bytes, mem_step_size) * mem_step_size);
+
+        if (bytes > 0) {
+            memory_info& current = this->getCurrentMemoryInfo();
+
+            // There is no memory cache in debug mode
+            if (!this->debug_mode) {
+
+                // FIXME: Add better checks for garbage collection
+                // Perhaps look at total memory available as a metric
+                if (this->checkMemoryLimit()) {
+                    this->garbageCollect();
+                }
+
+                free_iter iter = current.free_map.find(alloc_bytes);
+
+                if (iter != current.free_map.end() && !iter->second.empty()) {
+                    ptr = iter->second.back();
+                    iter->second.pop_back();
+                }
+
+            }
+
+            // Only comes here if buffer size not found or in debug mode
+            if (ptr == NULL) {
+                // Perform garbage collection if memory can not be allocated
+                try {
+                    ptr = this->nativeAlloc(alloc_bytes);
+                } catch (AfError &ex) {
+                    // If out of memory, run garbage collect and try again
+                    if (ex.getError() != AF_ERR_NO_MEM) throw;
+                    this->garbageCollect();
+                    ptr = this->nativeAlloc(alloc_bytes);
+                }
+                // Increment these two only when it succeeds to come here.
+                current.total_bytes += alloc_bytes;
+                current.total_buffers += 1;
+            }
+
+
+            locked_info info = {!user_lock, user_lock, alloc_bytes};
+            current.locked_map[ptr] = info;
+            current.lock_bytes += alloc_bytes;
+            current.lock_buffers++;
+        }
+        return ptr;
+    }
+
+    void unlock(void *ptr, bool user_unlock)
+    {
+        // Shortcut for empty arrays
+        if (!ptr) return;
+
+        lock_guard_t lock(this->memory_mutex);
+        memory_info& current = this->getCurrentMemoryInfo();
+
+        locked_iter iter = current.locked_map.find((void *)ptr);
+
+        // Pointer not found in locked map
+        if (iter == current.locked_map.end()) {
+            // Probably came from user, just free it
+            this->nativeFree(ptr);
+            return;
+        }
+
+        if (user_unlock) {
+            (iter->second).user_lock = false;
+        } else {
+            (iter->second).manager_lock = false;
+        }
+
+        // Return early if either one is locked
+        if ((iter->second).user_lock || (iter->second).manager_lock) return;
+
+        size_t bytes = iter->second.bytes;
+        current.lock_bytes -= iter->second.bytes;
+        current.lock_buffers--;
+
+        current.locked_map.erase(iter);
+
+        if (this->debug_mode) {
+            // Just free memory in debug mode
+            if ((iter->second).bytes > 0) {
+                this->nativeFree(iter->first);
+                current.total_buffers--;
+                current.total_bytes -= iter->second.bytes;
+            }
+        } else {
+            // In regular mode, move buffer to free map
+            free_iter fiter = current.free_map.find(bytes);
+            if (fiter != current.free_map.end()) {
+                // If found, push back
+                fiter->second.push_back(ptr);
+            } else {
+                // If not found, create new vector for this size
+                std::vector<void *> ptrs;
+                ptrs.push_back(ptr);
+                current.free_map[bytes] = ptrs;
+            }
+        }
+    }
+
+    void garbageCollect()
+    {
+        if (this->debug_mode) return;
+
+        lock_guard_t lock(this->memory_mutex);
+        memory_info& current = this->getCurrentMemoryInfo();
+
+        // Return if all buffers are locked
+        if (current.total_buffers == current.lock_buffers) return;
+
+        for (auto &kv : current.free_map) {
+            size_t num_ptrs = kv.second.size();
+            //Free memory by popping the last element
+            for (int n = num_ptrs-1; n >= 0; n--) {
+                this->nativeFree(kv.second[n]);
+                current.total_bytes -= kv.first;
+                current.total_buffers--;
+                kv.second.pop_back();
+            }
+        }
+        current.free_map.clear();
+    }
+
+
+    void printInfo(const char *msg, const int device)
+    {
+        lock_guard_t lock(this->memory_mutex);
+        const memory_info& current = this->getCurrentMemoryInfo();
+
+        std::cout << msg << std::endl;
+
+        static const std::string head("|     POINTER      |    SIZE    |  AF LOCK  | USER LOCK |");
+        static const std::string line(head.size(), '-');
+        std::cout << line << std::endl << head << std::endl << line << std::endl;
+
+        for(auto& kv : current.locked_map) {
+            std::string status_mngr("Yes");
+            std::string status_user("Unknown");
+            if(kv.second.user_lock)     status_user = "Yes";
+            else                        status_user = " No";
+
+            std::string unit = "KB";
+            double size = (double)(kv.second.bytes) / 1024;
+            if(size >= 1024) {
+                size = size / 1024;
+                unit = "MB";
+            }
+
+            std::cout << "|  " << std::right << std::setw(14) << kv.first << " "
+                << " | " << std::setw(7) << std::setprecision(4) << size << " " << unit
+                << " | " << std::setw(9) << status_mngr
+                << " | " << std::setw(9) << status_user
+                << " |"  << std::endl;
+        }
+
+        for(auto &kv : current.free_map) {
+
+            std::string status_mngr("No");
+            std::string status_user("No");
+
+            std::string unit = "KB";
+            double size = (double)(kv.first) / 1024;
+            if(size >= 1024) {
+                size = size / 1024;
+                unit = "MB";
+            }
+
+            for (auto &ptr : kv.second) {
+                std::cout << "|  " << std::right << std::setw(14) << ptr << " "
+                    << " | " << std::setw(7) << std::setprecision(4) << size << " " << unit
+                    << " | " << std::setw(9) << status_mngr
+                    << " | " << std::setw(9) << status_user
+                    << " |"  << std::endl;
+            }
+        }
+
+        std::cout << line << std::endl;
+    }
+
+    void bufferInfo(size_t *alloc_bytes, size_t *alloc_buffers,
+                    size_t *lock_bytes,  size_t *lock_buffers)
+    {
+        lock_guard_t lock(this->memory_mutex);
+        const memory_info& current = this->getCurrentMemoryInfo();
+        if (alloc_bytes   ) *alloc_bytes   = current.total_bytes;
+        if (alloc_buffers ) *alloc_buffers = current.total_buffers;
+        if (lock_bytes    ) *lock_bytes    = current.lock_bytes;
+        if (lock_buffers  ) *lock_buffers  = current.lock_buffers;
+    }
+
+    void userLock(const void *ptr)
+    {
+        memory_info& current = this->getCurrentMemoryInfo();
+
+        lock_guard_t lock(this->memory_mutex);
+
+        locked_iter iter = current.locked_map.find(const_cast<void *>(ptr));
+
+        if (iter != current.locked_map.end()) {
+            iter->second.user_lock = true;
+        } else {
+            locked_info info = {false,
+                true,
+                100}; //This number is not relevant
+
+            current.locked_map[(void *)ptr] = info;
+        }
+    }
+
+    void userUnlock(const void *ptr)
+    {
+        this->unlock(const_cast<void *>(ptr), true);
+    }
+
+    bool isUserLocked(const void *ptr)
+    {
+        memory_info& current = this->getCurrentMemoryInfo();
+        lock_guard_t lock(this->memory_mutex);
+        locked_iter iter = current.locked_map.find(const_cast<void *>(ptr));
+        if (iter != current.locked_map.end()) {
+            return iter->second.user_lock;
+        } else {
+            return false;
+        }
+    }
+
+    size_t getMemStepSize()
+    {
+        lock_guard_t lock(this->memory_mutex);
+        return this->mem_step_size;
+    }
+
+    size_t getMaxBytes()
+    {
+        lock_guard_t lock(this->memory_mutex);
+        return this->getCurrentMemoryInfo().max_bytes;
+    }
+
+    unsigned getMaxBuffers()
+    {
+        return this->max_buffers;
+    }
+
+    void setMemStepSize(size_t new_step_size)
+    {
+        lock_guard_t lock(this->memory_mutex);
+        this->mem_step_size = new_step_size;
+    }
+
+    inline void *nativeAlloc(const size_t bytes)
+    {
+        return static_cast<T*>(this)->nativeAlloc(bytes);
+    }
+
+    inline void nativeFree(void *ptr)
+    {
+        static_cast<T*>(this)->nativeFree(ptr);
+    }
+
+    virtual ~MemoryManager() {}
+
+    bool checkMemoryLimit()
+    {
+        const memory_info& current = this->getCurrentMemoryInfo();
+        return current.lock_bytes >= current.max_bytes || current.total_buffers >= this->max_buffers;
+    }
+
+    protected:
+    mutex_t memory_mutex;
+};
+
+}
diff --git a/src/backend/cpu/Array.cpp b/src/backend/cpu/Array.cpp
index 343b70f5d1..96d0345f6d 100644
--- a/src/backend/cpu/Array.cpp
+++ b/src/backend/cpu/Array.cpp
@@ -19,7 +19,6 @@
 #include <queue.hpp>
 #include <cstring>
 #include <cstddef>
-#include <MemoryManager.hpp>
 
 namespace cpu
 {
diff --git a/src/backend/cpu/memory.cpp b/src/backend/cpu/memory.cpp
index 25df00f4e2..8c246289a0 100644
--- a/src/backend/cpu/memory.cpp
+++ b/src/backend/cpu/memory.cpp
@@ -12,8 +12,6 @@
 #include <types.hpp>
 #include <platform.hpp>
 #include <queue.hpp>
-#include <memory>
-#include <MemoryManager.hpp>
 
 #ifndef AF_MEM_DEBUG
 #define AF_MEM_DEBUG 0
@@ -25,88 +23,34 @@
 
 namespace cpu
 {
-
-class MemoryManager  : public common::MemoryManager
-{
-    int getActiveDeviceId();
-    size_t getMaxMemorySize(int id);
-public:
-    MemoryManager();
-    void *nativeAlloc(const size_t bytes);
-    void nativeFree(void *ptr);
-    ~MemoryManager()
-    {
-        common::lock_guard_t lock(this->memory_mutex);
-        for (int n = 0; n < getDeviceCount(); n++) {
-            cpu::setDevice(n);
-            this->garbageCollect();
-        }
-    }
-};
-
-int MemoryManager::getActiveDeviceId()
-{
-    return cpu::getActiveDeviceId();
-}
-
-size_t MemoryManager::getMaxMemorySize(int id)
-{
-    return cpu::getDeviceMemorySize(id);
-}
-
-MemoryManager::MemoryManager() :
-    common::MemoryManager(getDeviceCount(), common::MAX_BUFFERS, AF_MEM_DEBUG || AF_CPU_MEM_DEBUG)
-{
-    this->setMaxMemorySize();
-}
-
-
-void *MemoryManager::nativeAlloc(const size_t bytes)
-{
-    void *ptr = malloc(bytes);
-    if (!ptr) AF_ERROR("Unable to allocate memory", AF_ERR_NO_MEM);
-    return ptr;
-}
-
-void MemoryManager::nativeFree(void *ptr)
-{
-    return free((void *)ptr);
-}
-
-static MemoryManager &getMemoryManager()
-{
-    static MemoryManager instance;
-    return instance;
-}
-
 void setMemStepSize(size_t step_bytes)
 {
-    getMemoryManager().setMemStepSize(step_bytes);
+    memoryManager().setMemStepSize(step_bytes);
 }
 
 size_t getMemStepSize(void)
 {
-    return getMemoryManager().getMemStepSize();
+    return memoryManager().getMemStepSize();
 }
 
 size_t getMaxBytes()
 {
-    return getMemoryManager().getMaxBytes();
+    return memoryManager().getMaxBytes();
 }
 
 unsigned getMaxBuffers()
 {
-    return getMemoryManager().getMaxBuffers();
+    return memoryManager().getMaxBuffers();
 }
 
 void garbageCollect()
 {
-    getMemoryManager().garbageCollect();
+    memoryManager().garbageCollect();
 }
 
 void printMemInfo(const char *msg, const int device)
 {
-    getMemoryManager().printInfo(msg, device);
+    memoryManager().printInfo(msg, device);
 }
 
 template<typename T>
@@ -115,10 +59,10 @@ T* memAlloc(const size_t &elements)
     T *ptr = nullptr;
 
     try {
-        ptr = (T *)getMemoryManager().alloc(elements * sizeof(T), false);
+        ptr = (T *)memoryManager().alloc(elements * sizeof(T), false);
     } catch(...) {
         getQueue().sync();
-        ptr = (T *)getMemoryManager().alloc(elements * sizeof(T), false);
+        ptr = (T *)memoryManager().alloc(elements * sizeof(T), false);
     }
     return ptr;
 }
@@ -128,10 +72,10 @@ void* memAllocUser(const size_t &bytes)
     void *ptr = nullptr;
 
     try {
-        ptr = getMemoryManager().alloc(bytes, true);
+        ptr = memoryManager().alloc(bytes, true);
     } catch(...) {
         getQueue().sync();
-        ptr = getMemoryManager().alloc(bytes, true);
+        ptr = memoryManager().alloc(bytes, true);
     }
     return ptr;
 }
@@ -139,52 +83,52 @@ void* memAllocUser(const size_t &bytes)
 template<typename T>
 void memFree(T *ptr)
 {
-    return getMemoryManager().unlock((void *)ptr, false);
+    return memoryManager().unlock((void *)ptr, false);
 }
 
 void memFreeUser(void *ptr)
 {
-    getMemoryManager().unlock((void *)ptr, true);
+    memoryManager().unlock((void *)ptr, true);
 }
 
 void memLock(const void *ptr)
 {
-    getMemoryManager().userLock((void *)ptr);
+    memoryManager().userLock((void *)ptr);
 }
 
 bool isLocked(const void *ptr)
 {
-    return getMemoryManager().isUserLocked((void *)ptr);
+    return memoryManager().isUserLocked((void *)ptr);
 }
 
 void memUnlock(const void *ptr)
 {
-    getMemoryManager().userUnlock((void *)ptr);
+    memoryManager().userUnlock((void *)ptr);
 }
 
 void deviceMemoryInfo(size_t *alloc_bytes, size_t *alloc_buffers,
                       size_t *lock_bytes,  size_t *lock_buffers)
 {
     getQueue().sync();
-    getMemoryManager().bufferInfo(alloc_bytes, alloc_buffers,
+    memoryManager().bufferInfo(alloc_bytes, alloc_buffers,
                                   lock_bytes,  lock_buffers);
 }
 
 template<typename T>
 T* pinnedAlloc(const size_t &elements)
 {
-    return (T *)getMemoryManager().alloc(elements * sizeof(T), false);
+    return (T *)memoryManager().alloc(elements * sizeof(T), false);
 }
 
 template<typename T>
 void pinnedFree(T* ptr)
 {
-    return getMemoryManager().unlock((void *)ptr, false);
+    return memoryManager().unlock((void *)ptr, false);
 }
 
 bool checkMemoryLimit()
 {
-    return getMemoryManager().checkMemoryLimit();
+    return memoryManager().checkMemoryLimit();
 }
 
 #define INSTANTIATE(T)                                      \
@@ -206,4 +150,45 @@ INSTANTIATE(uintl)
 INSTANTIATE(ushort)
 INSTANTIATE(short )
 
+MemoryManager::MemoryManager()
+    : common::MemoryManager<cpu::MemoryManager>(getDeviceCount(), common::MAX_BUFFERS,
+                                                AF_MEM_DEBUG || AF_CPU_MEM_DEBUG)
+{
+    this->setMaxMemorySize();
+}
+
+MemoryManager::~MemoryManager()
+{
+    common::lock_guard_t lock(this->memory_mutex);
+    for (int n = 0; n < cpu::getDeviceCount(); n++) {
+        try {
+            cpu::setDevice(n);
+            garbageCollect();
+        } catch(AfError err) {
+            continue; // Do not throw any errors while shutting down
+        }
+    }
+}
+
+int MemoryManager::getActiveDeviceId()
+{
+    return cpu::getActiveDeviceId();
+}
+
+size_t MemoryManager::getMaxMemorySize(int id)
+{
+    return cpu::getDeviceMemorySize(id);
+}
+
+void *MemoryManager::nativeAlloc(const size_t bytes)
+{
+    void *ptr = malloc(bytes);
+    if (!ptr) AF_ERROR("Unable to allocate memory", AF_ERR_NO_MEM);
+    return ptr;
+}
+
+void MemoryManager::nativeFree(void *ptr)
+{
+    return free((void *)ptr);
+}
 }
diff --git a/src/backend/cpu/memory.hpp b/src/backend/cpu/memory.hpp
index b2105c5af4..e85c89fc94 100644
--- a/src/backend/cpu/memory.hpp
+++ b/src/backend/cpu/memory.hpp
@@ -9,36 +9,48 @@
 #pragma once
 
 #include <af/defines.h>
+#include <common/MemoryManager.hpp>
 
 namespace cpu
 {
-    template<typename T> T* memAlloc(const size_t &elements);
-    void *memAllocUser(const size_t &bytes);
+template<typename T> T* memAlloc(const size_t &elements);
+void *memAllocUser(const size_t &bytes);
 
-    // Need these as 2 separate function and not a default argument
-    // This is because it is used as the deleter in shared pointer
-    // which cannot support default arguments
-    template<typename T> void memFree(T* ptr);
-    void memFreeUser(void* ptr);
+// Need these as 2 separate function and not a default argument
+// This is because it is used as the deleter in shared pointer
+// which cannot support default arguments
+template<typename T> void memFree(T* ptr);
+void memFreeUser(void* ptr);
 
-    void memLock(const void *ptr);
-    void memUnlock(const void *ptr);
-    bool isLocked(const void *ptr);
+void memLock(const void *ptr);
+void memUnlock(const void *ptr);
+bool isLocked(const void *ptr);
 
-    template<typename T> T* pinnedAlloc(const size_t &elements);
-    template<typename T> void pinnedFree(T* ptr);
+template<typename T> T* pinnedAlloc(const size_t &elements);
+template<typename T> void pinnedFree(T* ptr);
 
-    size_t getMaxBytes();
-    unsigned getMaxBuffers();
+size_t getMaxBytes();
+unsigned getMaxBuffers();
 
-    void deviceMemoryInfo(size_t *alloc_bytes, size_t *alloc_buffers,
-                          size_t *lock_bytes,  size_t *lock_buffers);
-    void garbageCollect();
-    void pinnedGarbageCollect();
+void deviceMemoryInfo(size_t *alloc_bytes, size_t *alloc_buffers,
+                      size_t *lock_bytes,  size_t *lock_buffers);
+void garbageCollect();
+void pinnedGarbageCollect();
 
-    void printMemInfo(const char *msg, const int device);
+void printMemInfo(const char *msg, const int device);
 
-    void setMemStepSize(size_t step_bytes);
-    size_t getMemStepSize(void);
-    bool checkMemoryLimit();
+void setMemStepSize(size_t step_bytes);
+size_t getMemStepSize(void);
+bool checkMemoryLimit();
+
+class MemoryManager : public common::MemoryManager<cpu::MemoryManager>
+{
+    public:
+        MemoryManager();
+        ~MemoryManager();
+        int getActiveDeviceId();
+        size_t getMaxMemorySize(int id);
+        void *nativeAlloc(const size_t bytes);
+        void nativeFree(void *ptr);
+};
 }
diff --git a/src/backend/cpu/platform.cpp b/src/backend/cpu/platform.cpp
index c8103bb93f..af694f90e1 100644
--- a/src/backend/cpu/platform.cpp
+++ b/src/backend/cpu/platform.cpp
@@ -257,6 +257,17 @@ bool& evalFlag()
     return flag;
 }
 
+MemoryManager& memoryManager()
+{
+    static std::once_flag flag;
+
+    DeviceManager& inst = DeviceManager::getInstance();
+
+    std::call_once(flag, [&]() { inst.memManager.reset(new MemoryManager()); });
+
+    return *(inst.memManager.get());
+}
+
 DeviceManager& DeviceManager::getInstance()
 {
     static DeviceManager my_instance;
diff --git a/src/backend/cpu/platform.hpp b/src/backend/cpu/platform.hpp
index 066b711f88..eeefbbec88 100644
--- a/src/backend/cpu/platform.hpp
+++ b/src/backend/cpu/platform.hpp
@@ -12,8 +12,9 @@
 #include <array>
 #include <algorithm>
 #include <iostream>
+#include <memory>
 #include <string>
-
+#include <memory.hpp>
 #include <queue.hpp>
 
 #if defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || defined(_M_IX86) || defined(_WIN64)
@@ -82,7 +83,6 @@ class CPUInfo {
 
 namespace cpu
 {
-
 int getBackend();
 
 std::string getDeviceInfo();
@@ -109,6 +109,8 @@ void sync(int device);
 
 bool& evalFlag();
 
+MemoryManager& memoryManager();
+
 class DeviceManager
 {
     public:
@@ -121,6 +123,8 @@ class DeviceManager
 
         friend queue& getQueue(int device);
 
+        friend MemoryManager& memoryManager();
+
         CPUInfo getCPUInfo() const;
 
     private:
@@ -137,6 +141,7 @@ class DeviceManager
         // Attributes
         const CPUInfo cinfo;
         std::array<queue, MAX_QUEUES> queues;
-};
 
+        std::unique_ptr<MemoryManager> memManager;
+};
 }
diff --git a/src/backend/cuda/Array.cpp b/src/backend/cuda/Array.cpp
index e94bcb787a..d34abd1d42 100644
--- a/src/backend/cuda/Array.cpp
+++ b/src/backend/cuda/Array.cpp
@@ -15,7 +15,6 @@
 #include <scalar.hpp>
 #include <memory.hpp>
 #include <platform.hpp>
-#include <MemoryManager.hpp>
 
 #include <cstddef>
 #include <memory>
diff --git a/src/backend/cuda/memory.cpp b/src/backend/cuda/memory.cpp
index 9b2eb07c44..f6261dc80a 100644
--- a/src/backend/cuda/memory.cpp
+++ b/src/backend/cuda/memory.cpp
@@ -7,103 +7,114 @@
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
 
+#include <cuda.h>
+#include <cuda_runtime_api.h>
+#include <cuda_runtime.h>
+#include <platform.hpp>
+#include <err_cuda.hpp>
 #include <memory.hpp>
 #include <util.hpp>
 #include <types.hpp>
 #include <dispatch.hpp>
-#include <memoryManager.hpp>
+
+#ifndef AF_MEM_DEBUG
+#define AF_MEM_DEBUG 0
+#endif
+
+#ifndef AF_CUDA_MEM_DEBUG
+#define AF_CUDA_MEM_DEBUG 0
+#endif
 
 namespace cuda
 {
-
 void setMemStepSize(size_t step_bytes)
 {
-    getMemoryManager().setMemStepSize(step_bytes);
+    memoryManager().setMemStepSize(step_bytes);
 }
 
 size_t getMemStepSize(void)
 {
-    return getMemoryManager().getMemStepSize();
+    return memoryManager().getMemStepSize();
 }
 
 size_t getMaxBytes()
 {
-    return getMemoryManager().getMaxBytes();
+    return memoryManager().getMaxBytes();
 }
 
 unsigned getMaxBuffers()
 {
-    return getMemoryManager().getMaxBuffers();
+    return memoryManager().getMaxBuffers();
 }
 
 void garbageCollect()
 {
-    getMemoryManager().garbageCollect();
+    memoryManager().garbageCollect();
 }
 
 void printMemInfo(const char *msg, const int device)
 {
-    getMemoryManager().printInfo(msg, device);
+    memoryManager().printInfo(msg, device);
 }
 
 template<typename T>
 T* memAlloc(const size_t &elements)
 {
-    return (T *)getMemoryManager().alloc(elements * sizeof(T), false);
+    return (T *)memoryManager().alloc(elements * sizeof(T), false);
 }
 
 void* memAllocUser(const size_t &bytes)
 {
-    return getMemoryManager().alloc(bytes, true);
+    return memoryManager().alloc(bytes, true);
 }
 template<typename T>
 void memFree(T *ptr)
 {
-    return getMemoryManager().unlock((void *)ptr, false);
+    return memoryManager().unlock((void *)ptr, false);
 }
 
 void memFreeUser(void *ptr)
 {
-    getMemoryManager().unlock((void *)ptr, true);
+    memoryManager().unlock((void *)ptr, true);
 }
 
 void memLock(const void *ptr)
 {
-    getMemoryManager().userLock((void *)ptr);
+    memoryManager().userLock((void *)ptr);
 }
 
 void memUnlock(const void *ptr)
 {
-    getMemoryManager().userUnlock((void *)ptr);
+    memoryManager().userUnlock((void *)ptr);
 }
 
 bool isLocked(const void *ptr)
 {
-    return getMemoryManager().isUserLocked((void *)ptr);
+    return memoryManager().isUserLocked((void *)ptr);
 }
 
 void deviceMemoryInfo(size_t *alloc_bytes, size_t *alloc_buffers,
                       size_t *lock_bytes,  size_t *lock_buffers)
 {
-    getMemoryManager().bufferInfo(alloc_bytes, alloc_buffers,
+    memoryManager().bufferInfo(alloc_bytes, alloc_buffers,
                                   lock_bytes,  lock_buffers);
 }
 
 template<typename T>
 T* pinnedAlloc(const size_t &elements)
 {
-    return (T *)getMemoryManagerPinned().alloc(elements * sizeof(T), false);
+    return (T *)pinnedMemoryManager().alloc(elements * sizeof(T), false);
 }
 
 template<typename T>
 void pinnedFree(T* ptr)
 {
-    return getMemoryManagerPinned().unlock((void *)ptr, false);
+    return pinnedMemoryManager().unlock((void *)ptr, false);
 }
 
 bool checkMemoryLimit()
 {
-    return getMemoryManager().checkMemoryLimit();
+    return memoryManager().checkMemoryLimit();
 }
 
 #define INSTANTIATE(T)                                      \
@@ -125,4 +136,85 @@ bool checkMemoryLimit()
     INSTANTIATE(short)
     INSTANTIATE(ushort)
 
+MemoryManager::MemoryManager()
+    : common::MemoryManager<cuda::MemoryManager>(getDeviceCount(), common::MAX_BUFFERS,
+                                                 AF_MEM_DEBUG || AF_CUDA_MEM_DEBUG)
+{
+    this->setMaxMemorySize();
+}
+
+MemoryManager::~MemoryManager()
+{
+    common::lock_guard_t lock(this->memory_mutex);
+    for (int n = 0; n < cuda::getDeviceCount(); n++) {
+        try {
+            cuda::setDevice(n);
+            garbageCollect();
+        } catch(AfError err) {
+            continue; // Do not throw any errors while shutting down
+        }
+    }
+}
+
+int MemoryManager::getActiveDeviceId()
+{
+    return cuda::getActiveDeviceId();
+}
+
+size_t MemoryManager::getMaxMemorySize(int id)
+{
+    return cuda::getDeviceMemorySize(id);
+}
+
+void *MemoryManager::nativeAlloc(const size_t bytes)
+{
+    void *ptr = NULL;
+    CUDA_CHECK(cudaMalloc(&ptr, bytes));
+    return ptr;
+}
+
+void MemoryManager::nativeFree(void *ptr)
+{
+    cudaError_t err = cudaFree(ptr);
+    if (err != cudaErrorCudartUnloading) {
+        CUDA_CHECK(err);
+    }
+}
+
+MemoryManagerPinned::MemoryManagerPinned()
+    : common::MemoryManager<MemoryManagerPinned>(1, common::MAX_BUFFERS,
+                                                 AF_MEM_DEBUG || AF_CUDA_MEM_DEBUG)
+{
+    this->setMaxMemorySize();
+}
+
+MemoryManagerPinned::~MemoryManagerPinned()
+{
+    garbageCollect();
+}
+
+int MemoryManagerPinned::getActiveDeviceId()
+{
+    return 0; // pinned uses a single vector
+}
+
+size_t MemoryManagerPinned::getMaxMemorySize(int id)
+{
+    return cuda::getHostMemorySize();
+}
+
+void *MemoryManagerPinned::nativeAlloc(const size_t bytes)
+{
+    void *ptr;
+    CUDA_CHECK(cudaMallocHost(&ptr, bytes));
+    return ptr;
+}
+
+void MemoryManagerPinned::nativeFree(void *ptr)
+{
+    cudaError_t err = cudaFreeHost(ptr);
+    if (err != cudaErrorCudartUnloading) {
+        CUDA_CHECK(err);
+    }
+}
 }
diff --git a/src/backend/cuda/memory.hpp b/src/backend/cuda/memory.hpp
index ab895168d4..9c0ff38503 100644
--- a/src/backend/cuda/memory.hpp
+++ b/src/backend/cuda/memory.hpp
@@ -9,37 +9,64 @@
 #pragma once
 
 #include <cstdlib>
+#include <common/MemoryManager.hpp>
 
 namespace cuda
 {
-    template<typename T> T* memAlloc(const size_t &elements);
-    void *memAllocUser(const size_t &bytes);
+template<typename T> T* memAlloc(const size_t &elements);
+void *memAllocUser(const size_t &bytes);
 
-    // Need these as 2 separate function and not a default argument
-    // This is because it is used as the deleter in shared pointer
-    // which cannot support default arguments
-    template<typename T> void memFree(T* ptr);
-    void memFreeUser(void* ptr);
+// Need these as 2 separate function and not a default argument
+// This is because it is used as the deleter in shared pointer
+// which cannot support default arguments
+template<typename T> void memFree(T* ptr);
+void memFreeUser(void* ptr);
 
-    void memLock(const void *ptr);
-    void memUnlock(const void *ptr);
-    bool isLocked(const void *ptr);
+void memLock(const void *ptr);
+void memUnlock(const void *ptr);
+bool isLocked(const void *ptr);
 
-    template<typename T> T* pinnedAlloc(const size_t &elements);
-    template<typename T> void pinnedFree(T* ptr);
+template<typename T> T* pinnedAlloc(const size_t &elements);
+template<typename T> void pinnedFree(T* ptr);
 
-    size_t getMaxBytes();
-    unsigned getMaxBuffers();
+size_t getMaxBytes();
+unsigned getMaxBuffers();
 
-    void deviceMemoryInfo(size_t *alloc_bytes, size_t *alloc_buffers,
-                          size_t *lock_bytes,  size_t *lock_buffers);
-    void garbageCollect();
-    void pinnedGarbageCollect();
+void deviceMemoryInfo(size_t *alloc_bytes, size_t *alloc_buffers,
+                      size_t *lock_bytes,  size_t *lock_buffers);
+void garbageCollect();
+void pinnedGarbageCollect();
 
-    void printMemInfo(const char *msg, const int device);
+void printMemInfo(const char *msg, const int device);
 
-    void setMemStepSize(size_t step_bytes);
-    size_t getMemStepSize(void);
+void setMemStepSize(size_t step_bytes);
+size_t getMemStepSize(void);
 
-    bool checkMemoryLimit();
+bool checkMemoryLimit();
+
+class MemoryManager : public common::MemoryManager<cuda::MemoryManager>
+{
+    public:
+        MemoryManager();
+        ~MemoryManager();
+        int getActiveDeviceId();
+        size_t getMaxMemorySize(int id);
+        void *nativeAlloc(const size_t bytes);
+        void nativeFree(void *ptr);
+};
+
+// CUDA Pinned Memory does not depend on device
+// So we pass 1 as numDevices to the constructor so that it creates 1 vector
+// of memory_info
+// When allocating and freeing, it doesn't really matter which device is active
+class MemoryManagerPinned : public common::MemoryManager<MemoryManagerPinned>
+{
+    public:
+        MemoryManagerPinned();
+        ~MemoryManagerPinned();
+        int getActiveDeviceId();
+        size_t getMaxMemorySize(int id);
+        void *nativeAlloc(const size_t bytes);
+        void nativeFree(void *ptr);
+};
 }
diff --git a/src/backend/cuda/memoryManager.cpp b/src/backend/cuda/memoryManager.cpp
deleted file mode 100644
index d160471814..0000000000
--- a/src/backend/cuda/memoryManager.cpp
+++ /dev/null
@@ -1,86 +0,0 @@
-/*******************************************************
- * Copyright (c) 2016, ArrayFire
- * All rights reserved.
- *
- * This file is distributed under 3-clause BSD license.
- * The complete license agreement can be obtained at:
- * http://arrayfire.com/licenses/BSD-3-Clause
- ********************************************************/
-
-#include <err_cuda.hpp>
-#include <memoryManager.hpp>
-
-#ifndef AF_MEM_DEBUG
-#define AF_MEM_DEBUG 0
-#endif
-
-#ifndef AF_CUDA_MEM_DEBUG
-#define AF_CUDA_MEM_DEBUG 0
-#endif
-
-namespace cuda
-{
-
-int MemoryManager::getActiveDeviceId()
-{
-    return cuda::getActiveDeviceId();
-}
-
-size_t MemoryManager::getMaxMemorySize(int id)
-{
-    return cuda::getDeviceMemorySize(id);
-}
-
-MemoryManager::MemoryManager() :
-    common::MemoryManager(getDeviceCount(), common::MAX_BUFFERS, AF_MEM_DEBUG || AF_CUDA_MEM_DEBUG)
-{
-    this->setMaxMemorySize();
-}
-
-void *MemoryManager::nativeAlloc(const size_t bytes)
-{
-    void *ptr = NULL;
-    CUDA_CHECK(cudaMalloc(&ptr, bytes));
-    return ptr;
-}
-
-void MemoryManager::nativeFree(void *ptr)
-{
-    cudaError_t err = cudaFree(ptr);
-    if (err != cudaErrorCudartUnloading) {
-        CUDA_CHECK(err);
-    }
-}
-
-int MemoryManagerPinned::getActiveDeviceId()
-{
-    return 0; // pinned uses a single vector
-}
-
-size_t MemoryManagerPinned::getMaxMemorySize(int id)
-{
-    return cuda::getHostMemorySize();
-}
-
-MemoryManagerPinned::MemoryManagerPinned() :
-    common::MemoryManager(1, common::MAX_BUFFERS, AF_MEM_DEBUG || AF_CUDA_MEM_DEBUG)
-{
-    this->setMaxMemorySize();
-}
-
-void *MemoryManagerPinned::nativeAlloc(const size_t bytes)
-{
-    void *ptr;
-    CUDA_CHECK(cudaMallocHost(&ptr, bytes));
-    return ptr;
-}
-
-void MemoryManagerPinned::nativeFree(void *ptr)
-{
-    cudaError_t err = cudaFreeHost(ptr);
-    if (err != cudaErrorCudartUnloading) {
-        CUDA_CHECK(err);
-    }
-}
-
-}
diff --git a/src/backend/cuda/memoryManager.hpp b/src/backend/cuda/memoryManager.hpp
deleted file mode 100644
index b22bd02c1f..0000000000
--- a/src/backend/cuda/memoryManager.hpp
+++ /dev/null
@@ -1,63 +0,0 @@
-/*******************************************************
- * Copyright (c) 2016, ArrayFire
- * All rights reserved.
- *
- * This file is distributed under 3-clause BSD license.
- * The complete license agreement can be obtained at:
- * http://arrayfire.com/licenses/BSD-3-Clause
- ********************************************************/
-
-#pragma once
-
-#include <cuda.h>
-#include <cuda_runtime_api.h>
-#include <cuda_runtime.h>
-
-#include <platform.hpp>
-#include <MemoryManager.hpp>
-
-namespace cuda
-{
-
-class MemoryManager  : public common::MemoryManager
-{
-    int getActiveDeviceId();
-    size_t getMaxMemorySize(int id);
-public:
-    MemoryManager();
-    void *nativeAlloc(const size_t bytes);
-    void nativeFree(void *ptr);
-    ~MemoryManager()
-    {
-        common::lock_guard_t lock(this->memory_mutex);
-        for (int n = 0; n < getDeviceCount(); n++) {
-            try {
-                cuda::setDevice(n);
-                this->garbageCollect();
-            } catch(AfError err) {
-                continue; // Do not throw any errors while shutting down
-            }
-        }
-    }
-};
-
-// CUDA Pinned Memory does not depend on device
-// So we pass 1 as numDevices to the constructor so that it creates 1 vector
-// of memory_info
-// When allocating and freeing, it doesn't really matter which device is active
-class MemoryManagerPinned  : public common::MemoryManager
-{
-    int getActiveDeviceId();
-    size_t getMaxMemorySize(int id);
-public:
-    MemoryManagerPinned();
-    void *nativeAlloc(const size_t bytes);
-    void nativeFree(void *ptr);
-    ~MemoryManagerPinned()
-    {
-        common::lock_guard_t lock(this->memory_mutex);
-        this->garbageCollect();
-    }
-};
-
-}
diff --git a/src/backend/cuda/platform.cpp b/src/backend/cuda/platform.cpp
index ca00ccf06c..fbfee69d34 100644
--- a/src/backend/cuda/platform.cpp
+++ b/src/backend/cuda/platform.cpp
@@ -24,7 +24,6 @@
 #include <err_cuda.hpp>
 #include <util.hpp>
 #include <host_memory.hpp>
-#include <memoryManager.hpp>
 #include <common/InteropManager.hpp>
 
 using namespace std;
@@ -386,28 +385,24 @@ DeviceManager& DeviceManager::getInstance()
     return my_instance;
 }
 
-MemoryManager &getMemoryManager()
+MemoryManager& memoryManager()
 {
     static std::once_flag flag;
 
     DeviceManager& inst = DeviceManager::getInstance();
 
-    std::call_once(flag, [&]() {
-                inst.memManager.reset(new cuda::MemoryManager());
-            });
+    std::call_once(flag, [&]() { inst.memManager.reset(new MemoryManager()); });
 
     return *(inst.memManager.get());
 }
 
-MemoryManagerPinned &getMemoryManagerPinned()
+MemoryManagerPinned& pinnedMemoryManager()
 {
     static std::once_flag flag;
 
     DeviceManager& inst = DeviceManager::getInstance();
 
-    std::call_once(flag, [&]() {
-                inst.pinnedMemManager.reset(new cuda::MemoryManagerPinned());
-            });
+    std::call_once(flag, [&]() { inst.pinnedMemManager.reset(new MemoryManagerPinned()); });
 
     return *(inst.pinnedMemManager.get());
 }
diff --git a/src/backend/cuda/platform.hpp b/src/backend/cuda/platform.hpp
index 5a025621d3..8440d58b81 100644
--- a/src/backend/cuda/platform.hpp
+++ b/src/backend/cuda/platform.hpp
@@ -14,7 +14,7 @@
 #include <memory>
 #include <vector>
 #include <string>
-
+#include <memory.hpp>
 #include <GraphicsResourceManager.hpp>
 #include <cufft.hpp>
 #include <cublas.hpp>
@@ -72,13 +72,11 @@ struct cudaDevice_t {
 
 bool& evalFlag();
 
-class MemoryManager;
-class MemoryManagerPinned;
 ///////////////////////// BEGIN Sub-Managers ///////////////////
 //
-MemoryManager& getMemoryManager();
+MemoryManager& memoryManager();
 
-MemoryManagerPinned& getMemoryManagerPinned();
+MemoryManagerPinned& pinnedMemoryManager();
 
 typedef common::InteropManager<GraphicsResourceManager, CGR_t> GraphicsManager;
 GraphicsManager& interopManager();
@@ -103,9 +101,9 @@ class DeviceManager
 
         static DeviceManager& getInstance();
 
-        friend MemoryManager& getMemoryManager();
+        friend MemoryManager& memoryManager();
 
-        friend MemoryManagerPinned& getMemoryManagerPinned();
+        friend MemoryManagerPinned& pinnedMemoryManager();
 
         friend GraphicsManager& interopManager();
 
diff --git a/src/backend/opencl/Array.cpp b/src/backend/opencl/Array.cpp
index 431de642cd..88355ee3dd 100644
--- a/src/backend/opencl/Array.cpp
+++ b/src/backend/opencl/Array.cpp
@@ -18,7 +18,6 @@
 #include <cstddef>
 #include <af/opencl.h>
 #include <util.hpp>
-#include <MemoryManager.hpp>
 
 using af::dim4;
 
diff --git a/src/backend/opencl/memory.cpp b/src/backend/opencl/memory.cpp
index 3542e3451a..e032f7bc6e 100644
--- a/src/backend/opencl/memory.cpp
+++ b/src/backend/opencl/memory.cpp
@@ -8,112 +8,118 @@
  ********************************************************/
 
 #include <memory.hpp>
-
-#include <memoryManager.hpp>
 #include <platform.hpp>
 #include <types.hpp>
+#include <err_opencl.hpp>
+
+#ifndef AF_MEM_DEBUG
+#define AF_MEM_DEBUG 0
+#endif
+
+#ifndef AF_OPENCL_MEM_DEBUG
+#define AF_OPENCL_MEM_DEBUG 0
+#endif
 
 namespace opencl
 {
-
 void setMemStepSize(size_t step_bytes)
 {
-    getMemoryManager().setMemStepSize(step_bytes);
+    memoryManager().setMemStepSize(step_bytes);
 }
 
 size_t getMemStepSize(void)
 {
-    return getMemoryManager().getMemStepSize();
+    return memoryManager().getMemStepSize();
 }
 
 size_t getMaxBytes()
 {
-    return getMemoryManager().getMaxBytes();
+    return memoryManager().getMaxBytes();
 }
 
 unsigned getMaxBuffers()
 {
-    return getMemoryManager().getMaxBuffers();
+    return memoryManager().getMaxBuffers();
 }
 
 void garbageCollect()
 {
-    getMemoryManager().garbageCollect();
+    memoryManager().garbageCollect();
 }
 
 void printMemInfo(const char *msg, const int device)
 {
-    getMemoryManager().printInfo(msg, device);
+    memoryManager().printInfo(msg, device);
 }
 
 template<typename T>
 T* memAlloc(const size_t &elements)
 {
-    return (T *)getMemoryManager().alloc(elements * sizeof(T), false);
+    return (T *)memoryManager().alloc(elements * sizeof(T), false);
 }
 
 void* memAllocUser(const size_t &bytes)
 {
-    return getMemoryManager().alloc(bytes, true);
+    return memoryManager().alloc(bytes, true);
 }
 template<typename T>
 void memFree(T *ptr)
 {
-    return getMemoryManager().unlock((void *)ptr, false);
+    return memoryManager().unlock((void *)ptr, false);
 }
 
 void memFreeUser(void *ptr)
 {
-    getMemoryManager().unlock((void *)ptr, true);
+    memoryManager().unlock((void *)ptr, true);
 }
 
 cl::Buffer *bufferAlloc(const size_t &bytes)
 {
-    return (cl::Buffer *)getMemoryManager().alloc(bytes, false);
+    return (cl::Buffer *)memoryManager().alloc(bytes, false);
 }
 
 void bufferFree(cl::Buffer *buf)
 {
-    return getMemoryManager().unlock((void *)buf, false);
+    return memoryManager().unlock((void *)buf, false);
 }
 
 void memLock(const void *ptr)
 {
-    getMemoryManager().userLock((void *)ptr);
+    memoryManager().userLock((void *)ptr);
 }
 
 void memUnlock(const void *ptr)
 {
-    getMemoryManager().userUnlock((void *)ptr);
+    memoryManager().userUnlock((void *)ptr);
 }
 
 bool isLocked(const void *ptr)
 {
-    return getMemoryManager().isUserLocked((void *)ptr);
+    return memoryManager().isUserLocked((void *)ptr);
 }
 
 void deviceMemoryInfo(size_t *alloc_bytes, size_t *alloc_buffers,
                       size_t *lock_bytes,  size_t *lock_buffers)
 {
-    getMemoryManager().bufferInfo(alloc_bytes, alloc_buffers,
+    memoryManager().bufferInfo(alloc_bytes, alloc_buffers,
                                   lock_bytes,  lock_buffers);
 }
 
 template<typename T>
 T* pinnedAlloc(const size_t &elements)
 {
-    return (T *)getMemoryManagerPinned().alloc(elements * sizeof(T), false);
+    return (T *)pinnedMemoryManager().alloc(elements * sizeof(T), false);
 }
 
 template<typename T>
 void pinnedFree(T* ptr)
 {
-    return getMemoryManagerPinned().unlock((void *)ptr, false);
+    return pinnedMemoryManager().unlock((void *)ptr, false);
 }
 
 bool checkMemoryLimit()
 {
-    return getMemoryManager().checkMemoryLimit();
+    return memoryManager().checkMemoryLimit();
 }
 
 #define INSTANTIATE(T)                                      \
@@ -134,4 +140,100 @@ bool checkMemoryLimit()
     INSTANTIATE(uintl)
     INSTANTIATE(short)
     INSTANTIATE(ushort)
+
+MemoryManager::MemoryManager()
+    : common::MemoryManager<opencl::MemoryManager>(getDeviceCount(), common::MAX_BUFFERS,
+                                                   AF_MEM_DEBUG || AF_OPENCL_MEM_DEBUG)
+{
+    this->setMaxMemorySize();
+}
+
+MemoryManager::~MemoryManager()
+{
+    common::lock_guard_t lock(this->memory_mutex);
+    for (int n = 0; n < opencl::getDeviceCount(); n++) {
+        try {
+            opencl::setDevice(n);
+            garbageCollect();
+        } catch(AfError err) {
+            continue; // Do not throw any errors while shutting down
+        }
+    }
+}
+
+int MemoryManager::getActiveDeviceId()
+{
+    return opencl::getActiveDeviceId();
+}
+
+size_t MemoryManager::getMaxMemorySize(int id)
+{
+    return opencl::getDeviceMemorySize(id);
+}
+
+void *MemoryManager::nativeAlloc(const size_t bytes)
+{
+    return (void *)(new cl::Buffer(getContext(), CL_MEM_READ_WRITE, bytes));
+}
+
+void MemoryManager::nativeFree(void *ptr)
+{
+    delete (cl::Buffer *)ptr;
+}
+
+MemoryManagerPinned::MemoryManagerPinned()
+    : common::MemoryManager<MemoryManagerPinned>(getDeviceCount(), common::MAX_BUFFERS,
+                                                 AF_MEM_DEBUG || AF_OPENCL_MEM_DEBUG),
+    pinnedMaps(getDeviceCount())
+{
+    this->setMaxMemorySize();
+}
+
+MemoryManagerPinned::~MemoryManagerPinned()
+{
+    common::lock_guard_t lock(this->memory_mutex);
+    for (int n = 0; n < getDeviceCount(); n++) {
+        opencl::setDevice(n);
+        garbageCollect();
+        auto currIterator = pinnedMaps[n].begin();
+        auto endIterator  = pinnedMaps[n].end();
+        while (currIterator != endIterator) {
+            delete currIterator->second;
+            pinnedMaps[n].erase(currIterator++);
+        }
+    }
+}
+
+int MemoryManagerPinned::getActiveDeviceId()
+{
+    return opencl::getActiveDeviceId();
+}
+
+size_t MemoryManagerPinned::getMaxMemorySize(int id)
+{
+    return opencl::getDeviceMemorySize(id);
+}
+
+void *MemoryManagerPinned::nativeAlloc(const size_t bytes)
+{
+    void *ptr = NULL;
+    cl::Buffer* buf = new cl::Buffer(getContext(), CL_MEM_ALLOC_HOST_PTR, bytes);
+    ptr = getQueue().enqueueMapBuffer(*buf, true, CL_MAP_READ | CL_MAP_WRITE, 0, bytes);
+    pinnedMaps[opencl::getActiveDeviceId()].emplace(ptr, buf);
+    return ptr;
+}
+
+void MemoryManagerPinned::nativeFree(void *ptr)
+{
+    int n = opencl::getActiveDeviceId();
+    auto map = pinnedMaps[n];
+    auto iter = map.find(ptr);
+
+    if (iter != map.end()) {
+        cl::Buffer* buf = map[ptr];
+        getQueue().enqueueUnmapMemObject(*buf, ptr);
+        delete buf;
+        map.erase(iter);
+    }
+}
 }
diff --git a/src/backend/opencl/memory.hpp b/src/backend/opencl/memory.hpp
index 823fd4581f..c6af6dfa1d 100644
--- a/src/backend/opencl/memory.hpp
+++ b/src/backend/opencl/memory.hpp
@@ -9,45 +9,71 @@
 #pragma once
 
 #include <cstdlib>
+#include <map>
+#include <vector>
+#include <common/MemoryManager.hpp>
 
 namespace cl
 {
-class Buffer;
+class Buffer; //Forward declaration of cl::Buffer from CL/cl2.hpp
 }
 
 namespace opencl
 {
+cl::Buffer *bufferAlloc(const size_t &bytes);
+void bufferFree(cl::Buffer *buf);
 
-    cl::Buffer *bufferAlloc(const size_t &bytes);
-    void bufferFree(cl::Buffer *buf);
+template<typename T> T* memAlloc(const size_t &elements);
+void *memAllocUser(const size_t &bytes);
 
-    template<typename T> T* memAlloc(const size_t &elements);
-    void *memAllocUser(const size_t &bytes);
+// Need these as 2 separate function and not a default argument
+// This is because it is used as the deleter in shared pointer
+// which cannot support default arguments
+template<typename T> void memFree(T* ptr);
+void memFreeUser(void* ptr);
 
-    // Need these as 2 separate function and not a default argument
-    // This is because it is used as the deleter in shared pointer
-    // which cannot support default arguments
-    template<typename T> void memFree(T* ptr);
-    void memFreeUser(void* ptr);
+void memLock(const void *ptr);
+void memUnlock(const void *ptr);
+bool isLocked(const void *ptr);
 
-    void memLock(const void *ptr);
-    void memUnlock(const void *ptr);
-    bool isLocked(const void *ptr);
+template<typename T> T* pinnedAlloc(const size_t &elements);
+template<typename T> void pinnedFree(T* ptr);
 
-    template<typename T> T* pinnedAlloc(const size_t &elements);
-    template<typename T> void pinnedFree(T* ptr);
+size_t getMaxBytes();
+unsigned getMaxBuffers();
 
-    size_t getMaxBytes();
-    unsigned getMaxBuffers();
+void deviceMemoryInfo(size_t *alloc_bytes, size_t *alloc_buffers,
+                      size_t *lock_bytes,  size_t *lock_buffers);
+void garbageCollect();
+void pinnedGarbageCollect();
 
-    void deviceMemoryInfo(size_t *alloc_bytes, size_t *alloc_buffers,
-                          size_t *lock_bytes,  size_t *lock_buffers);
-    void garbageCollect();
-    void pinnedGarbageCollect();
+void printMemInfo(const char *msg, const int device);
 
-    void printMemInfo(const char *msg, const int device);
+void setMemStepSize(size_t step_bytes);
+size_t getMemStepSize(void);
+bool checkMemoryLimit();
 
-    void setMemStepSize(size_t step_bytes);
-    size_t getMemStepSize(void);
-    bool checkMemoryLimit();
+class MemoryManager : public common::MemoryManager<opencl::MemoryManager>
+{
+    public:
+        MemoryManager();
+        ~MemoryManager();
+        int getActiveDeviceId();
+        size_t getMaxMemorySize(int id);
+        void *nativeAlloc(const size_t bytes);
+        void nativeFree(void *ptr);
+};
+
+class MemoryManagerPinned : public common::MemoryManager<MemoryManagerPinned>
+{
+    public:
+        MemoryManagerPinned();
+        ~MemoryManagerPinned();
+        int getActiveDeviceId();
+        size_t getMaxMemorySize(int id);
+        void *nativeAlloc(const size_t bytes);
+        void nativeFree(void *ptr);
+    private:
+        std::vector< std::map<void *, cl::Buffer*> > pinnedMaps;
+};
 }
diff --git a/src/backend/opencl/memoryManager.cpp b/src/backend/opencl/memoryManager.cpp
deleted file mode 100644
index c329faef5c..0000000000
--- a/src/backend/opencl/memoryManager.cpp
+++ /dev/null
@@ -1,86 +0,0 @@
-/*******************************************************
- * Copyright (c) 2016, ArrayFire
- * All rights reserved.
- *
- * This file is distributed under 3-clause BSD license.
- * The complete license agreement can be obtained at:
- * http://arrayfire.com/licenses/BSD-3-Clause
- ********************************************************/
-
-#include <memoryManager.hpp>
-
-#ifndef AF_MEM_DEBUG
-#define AF_MEM_DEBUG 0
-#endif
-
-#ifndef AF_OPENCL_MEM_DEBUG
-#define AF_OPENCL_MEM_DEBUG 0
-#endif
-
-namespace opencl
-{
-
-int MemoryManager::getActiveDeviceId()
-{
-    return opencl::getActiveDeviceId();
-}
-
-size_t MemoryManager::getMaxMemorySize(int id)
-{
-    return opencl::getDeviceMemorySize(id);
-}
-
-MemoryManager::MemoryManager() :
-    common::MemoryManager(getDeviceCount(), common::MAX_BUFFERS, AF_MEM_DEBUG || AF_OPENCL_MEM_DEBUG)
-{
-    this->setMaxMemorySize();
-}
-
-void *MemoryManager::nativeAlloc(const size_t bytes)
-{
-    return (void *)(new cl::Buffer(getContext(), CL_MEM_READ_WRITE, bytes));
-}
-
-void MemoryManager::nativeFree(void *ptr)
-{
-    delete (cl::Buffer *)ptr;
-}
-
-int MemoryManagerPinned::getActiveDeviceId()
-{
-    return opencl::getActiveDeviceId();
-}
-
-size_t MemoryManagerPinned::getMaxMemorySize(int id)
-{
-    return opencl::getDeviceMemorySize(id);
-}
-
-MemoryManagerPinned::MemoryManagerPinned() :
-    common::MemoryManager(getDeviceCount(), common::MAX_BUFFERS, AF_MEM_DEBUG || AF_OPENCL_MEM_DEBUG),
-    pinned_maps(getDeviceCount())
-{
-    this->setMaxMemorySize();
-}
-
-void *MemoryManagerPinned::nativeAlloc(const size_t bytes)
-{
-    void *ptr = NULL;
-    cl::Buffer buf= cl::Buffer(getContext(), CL_MEM_ALLOC_HOST_PTR, bytes);
-    ptr = getQueue().enqueueMapBuffer(buf, true, CL_MAP_READ | CL_MAP_WRITE, 0, bytes);
-    pinned_maps[opencl::getActiveDeviceId()][ptr] = buf;
-    return ptr;
-}
-
-void MemoryManagerPinned::nativeFree(void *ptr)
-{
-    int n = opencl::getActiveDeviceId();
-    auto iter = pinned_maps[n].find(ptr);
-
-    if (iter != pinned_maps[n].end()) {
-        getQueue().enqueueUnmapMemObject(pinned_maps[n][ptr], ptr);
-        pinned_maps[n].erase(iter);
-    }
-}
-
-}
diff --git a/src/backend/opencl/memoryManager.hpp b/src/backend/opencl/memoryManager.hpp
deleted file mode 100644
index eab26ed235..0000000000
--- a/src/backend/opencl/memoryManager.hpp
+++ /dev/null
@@ -1,68 +0,0 @@
-/*******************************************************
- * Copyright (c) 2014, ArrayFire
- * All rights reserved.
- *
- * This file is distributed under 3-clause BSD license.
- * The complete license agreement can be obtained at:
- * http://arrayfire.com/licenses/BSD-3-Clause
- ********************************************************/
-
-#pragma once
-
-#include <platform.hpp>
-#include <MemoryManager.hpp>
-
-#include <map>
-
-namespace opencl
-{
-
-class MemoryManager  : public common::MemoryManager
-{
-    int getActiveDeviceId();
-    size_t getMaxMemorySize(int id);
-public:
-    MemoryManager();
-    void *nativeAlloc(const size_t bytes);
-    void nativeFree(void *ptr);
-    ~MemoryManager()
-    {
-        common::lock_guard_t lock(this->memory_mutex);
-        for (int n = 0; n < getDeviceCount(); n++) {
-            opencl::setDevice(n);
-            this->garbageCollect();
-        }
-    }
-};
-
-class MemoryManagerPinned  : public common::MemoryManager
-{
-    std::vector<
-        std::map<void *, cl::Buffer>
-        > pinned_maps;
-    int getActiveDeviceId();
-    size_t getMaxMemorySize(int id);
-
-public:
-
-    MemoryManagerPinned();
-
-    void *nativeAlloc(const size_t bytes);
-    void nativeFree(void *ptr);
-
-    ~MemoryManagerPinned()
-    {
-        common::lock_guard_t lock(this->memory_mutex);
-        for (int n = 0; n < getDeviceCount(); n++) {
-            opencl::setDevice(n);
-            this->garbageCollect();
-            auto pinned_curr_iter = pinned_maps[n].begin();
-            auto pinned_end_iter  = pinned_maps[n].end();
-            while (pinned_curr_iter != pinned_end_iter) {
-                pinned_maps[n].erase(pinned_curr_iter++);
-            }
-        }
-    }
-};
-
-}
diff --git a/src/backend/opencl/platform.cpp b/src/backend/opencl/platform.cpp
index 31b3abafa3..295fcccad5 100644
--- a/src/backend/opencl/platform.cpp
+++ b/src/backend/opencl/platform.cpp
@@ -29,7 +29,6 @@
 #include <err_opencl.hpp>
 #include <host_memory.hpp>
 #include <common/InteropManager.hpp>
-#include <memoryManager.hpp>
 #include <platform.hpp>
 #include <util.hpp>
 #include <version.hpp>
@@ -839,28 +838,24 @@ bool& evalFlag()
     return flag;
 }
 
-MemoryManager& getMemoryManager()
+MemoryManager& memoryManager()
 {
     static std::once_flag flag;
 
     DeviceManager& inst = DeviceManager::getInstance();
 
-    std::call_once(flag, [&]() {
-                inst.memManager.reset(new MemoryManager());
-            });
+    std::call_once(flag, [&]() { inst.memManager.reset(new MemoryManager()); });
 
     return *(inst.memManager.get());
 }
 
-MemoryManagerPinned& getMemoryManagerPinned()
+MemoryManagerPinned& pinnedMemoryManager()
 {
     static std::once_flag flag;
 
     DeviceManager& inst = DeviceManager::getInstance();
 
-    std::call_once(flag, [&]() {
-                inst.pinnedMemManager.reset(new MemoryManagerPinned());
-            });
+    std::call_once(flag, [&]() { inst.pinnedMemManager.reset(new MemoryManagerPinned()); });
 
     return *(inst.pinnedMemManager.get());
 }
diff --git a/src/backend/opencl/platform.hpp b/src/backend/opencl/platform.hpp
index d94d60ae43..71c2ae0c56 100644
--- a/src/backend/opencl/platform.hpp
+++ b/src/backend/opencl/platform.hpp
@@ -26,6 +26,7 @@
 #include <vector>
 #include <string>
 
+#include <memory.hpp>
 #include <GraphicsResourceManager.hpp>
 #include <clfft.hpp>
 
@@ -82,15 +83,11 @@ int getActivePlatform();
 
 bool& evalFlag();
 
-// Forward Declarations
-class MemoryManager;
-class MemoryManagerPinned;
-
 ///////////////////////// BEGIN Sub-Managers ///////////////////
 //
-MemoryManager &getMemoryManager();
+MemoryManager& memoryManager();
 
-MemoryManagerPinned& getMemoryManagerPinned();
+MemoryManagerPinned& pinnedMemoryManager();
 
 typedef common::InteropManager<GraphicsResourceManager, CGR_t> GraphicsManager;
 GraphicsManager& interopManager();
@@ -102,9 +99,9 @@ FFTManager& clfftManager();
 
 class DeviceManager
 {
-    friend MemoryManager &getMemoryManager();
+    friend MemoryManager& memoryManager();
 
-    friend MemoryManagerPinned& getMemoryManagerPinned();
+    friend MemoryManagerPinned& pinnedMemoryManager();
 
     friend GraphicsManager& interopManager();
 

From 391d0e341f40b9762211bed8775d4e4c1b08c204 Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Tue, 17 Jan 2017 19:40:23 +0530
Subject: [PATCH 27/82] Fix CRTP classes of graphics, fft, blas, solve, sparse

---
 src/backend/cuda/GraphicsResourceManager.hpp |  2 ++
 src/backend/cuda/blas.cpp                    |  6 ++--
 src/backend/cuda/cholesky.cu                 |  4 +--
 src/backend/cuda/cublas.hpp                  |  2 --
 src/backend/cuda/cufft.cpp                   |  4 +--
 src/backend/cuda/cusolverDn.hpp              |  2 --
 src/backend/cuda/cusparse.hpp                |  3 --
 src/backend/cuda/fft.cpp                     |  2 +-
 src/backend/cuda/hist_graphics.cpp           |  4 +--
 src/backend/cuda/image.cpp                   |  4 +--
 src/backend/cuda/lu.cu                       |  4 +--
 src/backend/cuda/platform.cpp                | 19 ++++++------
 src/backend/cuda/platform.hpp                | 32 +++++++++-----------
 src/backend/cuda/plot.cpp                    |  4 +--
 src/backend/cuda/qr.cu                       | 10 +++---
 src/backend/cuda/solve.cu                    | 16 +++++-----
 src/backend/cuda/sparse.cu                   | 30 +++++++++---------
 src/backend/cuda/sparse_blas.cpp             |  4 +--
 src/backend/cuda/surface.cpp                 |  4 +--
 src/backend/cuda/svd.cu                      |  4 +--
 src/backend/cuda/vector_field.cpp            |  4 +--
 src/backend/opencl/clfft.cpp                 |  2 +-
 src/backend/opencl/fft.cpp                   |  2 +-
 src/backend/opencl/hist_graphics.cpp         |  4 +--
 src/backend/opencl/image.cpp                 |  4 +--
 src/backend/opencl/platform.cpp              |  6 ++--
 src/backend/opencl/platform.hpp              | 14 ++++-----
 src/backend/opencl/plot.cpp                  |  4 +--
 src/backend/opencl/surface.cpp               |  4 +--
 src/backend/opencl/vector_field.cpp          |  4 +--
 30 files changed, 98 insertions(+), 110 deletions(-)

diff --git a/src/backend/cuda/GraphicsResourceManager.hpp b/src/backend/cuda/GraphicsResourceManager.hpp
index b897d500c6..6f8b39fd8e 100644
--- a/src/backend/cuda/GraphicsResourceManager.hpp
+++ b/src/backend/cuda/GraphicsResourceManager.hpp
@@ -7,6 +7,8 @@
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
 
+#pragma once
+
 #if defined(WITH_GRAPHICS)
 #if defined(OS_WIN)
 #include <windows.h>
diff --git a/src/backend/cuda/blas.cpp b/src/backend/cuda/blas.cpp
index 46e4bfa8fb..921fe685ea 100644
--- a/src/backend/cuda/blas.cpp
+++ b/src/backend/cuda/blas.cpp
@@ -171,7 +171,7 @@ Array<T> matmul(const Array<T> &lhs, const Array<T> &rhs,
     if(rDims[bColDim] == 1) {
         N = lDims[aColDim];
         CUBLAS_CHECK(gemv_func<T>()(
-                         cublasHandle(),
+                         blasHandle(),
                          lOpts,
                          lDims[0],
                          lDims[1],
@@ -182,7 +182,7 @@ Array<T> matmul(const Array<T> &lhs, const Array<T> &rhs,
                          out.get(), 1));
     } else {
         CUBLAS_CHECK(gemm_func<T>()(
-                         cublasHandle(),
+                         blasHandle(),
                          lOpts,
                          rOpts,
                          M, N, K,
@@ -224,7 +224,7 @@ void trsm(const Array<T> &lhs, Array<T> &rhs, af_mat_prop trans,
     dim4 rStrides = rhs.strides();
 
     CUBLAS_CHECK(trsm_func<T>()(
-                     cublasHandle(),
+                     blasHandle(),
                      is_left  ? CUBLAS_SIDE_LEFT : CUBLAS_SIDE_RIGHT,
                      is_upper ? CUBLAS_FILL_MODE_UPPER : CUBLAS_FILL_MODE_LOWER,
                      toCblasTranspose(trans),
diff --git a/src/backend/cuda/cholesky.cu b/src/backend/cuda/cholesky.cu
index 53df19b67f..dc7a239f8b 100644
--- a/src/backend/cuda/cholesky.cu
+++ b/src/backend/cuda/cholesky.cu
@@ -110,7 +110,7 @@ int cholesky_inplace(Array<T> &in, const bool is_upper)
     if(is_upper)
         uplo = CUBLAS_FILL_MODE_UPPER;
 
-    CUSOLVER_CHECK(potrf_buf_func<T>()(cusolverDnHandle(),
+    CUSOLVER_CHECK(potrf_buf_func<T>()(solverDnHandle(),
                                        uplo,
                                        N,
                                        in.get(), in.strides()[1],
@@ -119,7 +119,7 @@ int cholesky_inplace(Array<T> &in, const bool is_upper)
     T *workspace = memAlloc<T>(lwork);
     int *d_info = memAlloc<int>(1);
 
-    CUSOLVER_CHECK(potrf_func<T>()(cusolverDnHandle(),
+    CUSOLVER_CHECK(potrf_func<T>()(solverDnHandle(),
                                    uplo,
                                    N,
                                    in.get(), in.strides()[1],
diff --git a/src/backend/cuda/cublas.hpp b/src/backend/cuda/cublas.hpp
index 5e5dd40922..994cb92023 100644
--- a/src/backend/cuda/cublas.hpp
+++ b/src/backend/cuda/cublas.hpp
@@ -41,6 +41,4 @@ class cublasHandle : public common::MatrixAlgebraHandle<cublasHandle, BlasHandle
             cublasDestroy(handle);
         }
 };
-
-typedef common::MatrixAlgebraHandle<cublasHandle, BlasHandle> BlasHandleWrapper;
 }
diff --git a/src/backend/cuda/cufft.cpp b/src/backend/cuda/cufft.cpp
index ac743f47f8..b185486158 100644
--- a/src/backend/cuda/cufft.cpp
+++ b/src/backend/cuda/cufft.cpp
@@ -13,7 +13,6 @@
 
 namespace cuda
 {
-
 const char * _cufftGetResultString(cufftResult res)
 {
     switch (res)
@@ -112,7 +111,7 @@ PlanType findPlan(int rank, int *n,
     sprintf(key_str_temp, "%d:%d", (int)type, batch);
     key_string.append(std::string(key_str_temp));
 
-    FFTManager &planner = cuda::cufftManager();
+    PlanCache &planner = cuda::fftManager();
 
     int planIndex = planner.find(key_string);
 
@@ -141,5 +140,4 @@ PlanType findPlan(int rank, int *n,
 
     return retVal;
 }
-
 }
diff --git a/src/backend/cuda/cusolverDn.hpp b/src/backend/cuda/cusolverDn.hpp
index 67592e67f0..dd42094a0c 100644
--- a/src/backend/cuda/cusolverDn.hpp
+++ b/src/backend/cuda/cusolverDn.hpp
@@ -46,6 +46,4 @@ class cusolverDnHandle : public common::MatrixAlgebraHandle<cusolverDnHandle, So
             cusolverDnDestroy(handle);
         }
 };
-
-typedef common::MatrixAlgebraHandle<cusolverDnHandle, SolveHandle> SolveHandleWrapper;
 }
diff --git a/src/backend/cuda/cusparse.hpp b/src/backend/cuda/cusparse.hpp
index 3bbdec81d1..bb4237fb6a 100644
--- a/src/backend/cuda/cusparse.hpp
+++ b/src/backend/cuda/cusparse.hpp
@@ -39,7 +39,4 @@ class cusparseHandle : public common::MatrixAlgebraHandle<cusparseHandle, Sparse
             cusparseDestroy(handle);
         }
 };
-
-typedef common::MatrixAlgebraHandle<cusparseHandle, SparseHandle> SparseHandleWrapper;
-
 }
diff --git a/src/backend/cuda/fft.cpp b/src/backend/cuda/fft.cpp
index 19a753421a..ca41896e29 100644
--- a/src/backend/cuda/fft.cpp
+++ b/src/backend/cuda/fft.cpp
@@ -23,7 +23,7 @@ namespace cuda
 {
 void setFFTPlanCacheSize(size_t numPlans)
 {
-    cufftManager().maxCacheSize(numPlans);
+    fftManager().maxCacheSize(numPlans);
 }
 
 template<typename T>
diff --git a/src/backend/cuda/hist_graphics.cpp b/src/backend/cuda/hist_graphics.cpp
index f74e03c23a..6367208c5c 100644
--- a/src/backend/cuda/hist_graphics.cpp
+++ b/src/backend/cuda/hist_graphics.cpp
@@ -13,7 +13,7 @@
 #include <hist_graphics.hpp>
 #include <err_cuda.hpp>
 #include <debug_cuda.hpp>
-#include <common/InteropManager.hpp>
+#include <GraphicsResourceManager.hpp>
 
 namespace cuda
 {
@@ -25,7 +25,7 @@ void copy_histogram(const Array<T> &data, const forge::Histogram* hist)
     if(DeviceManager::checkGraphicsInteropCapability()) {
         const T *d_P = data.get();
 
-        GraphicsManager& intrpMngr = interopManager();
+        GraphicsResourceManager& intrpMngr = interopManager();
 
         cudaGraphicsResource_t *resources = intrpMngr.getBufferResource(hist);
         // Map resource. Copy data to VBO. Unmap resource.
diff --git a/src/backend/cuda/image.cpp b/src/backend/cuda/image.cpp
index de9d0efa9c..17d0f12b0f 100644
--- a/src/backend/cuda/image.cpp
+++ b/src/backend/cuda/image.cpp
@@ -16,7 +16,7 @@
 #include <image.hpp>
 #include <err_cuda.hpp>
 #include <debug_cuda.hpp>
-#include <common/InteropManager.hpp>
+#include <GraphicsResourceManager.hpp>
 
 using af::dim4;
 
@@ -28,7 +28,7 @@ template<typename T>
 void copy_image(const Array<T> &in, const forge::Image* image)
 {
     if(DeviceManager::checkGraphicsInteropCapability()) {
-        GraphicsManager& intrpMngr = interopManager();
+        GraphicsResourceManager& intrpMngr = interopManager();
 
         cudaGraphicsResource_t *resources = intrpMngr.getBufferResource(image);
 
diff --git a/src/backend/cuda/lu.cu b/src/backend/cuda/lu.cu
index 79bbd5029e..104bc9025f 100644
--- a/src/backend/cuda/lu.cu
+++ b/src/backend/cuda/lu.cu
@@ -129,7 +129,7 @@ Array<int> lu_inplace(Array<T> &in, const bool convert_pivot)
 
     int lwork = 0;
 
-    CUSOLVER_CHECK(getrf_buf_func<T>()(cusolverDnHandle(),
+    CUSOLVER_CHECK(getrf_buf_func<T>()(solverDnHandle(),
                                        M, N,
                                        in.get(), in.strides()[1],
                                        &lwork));
@@ -137,7 +137,7 @@ Array<int> lu_inplace(Array<T> &in, const bool convert_pivot)
     T *workspace = memAlloc<T>(lwork);
     int *info = memAlloc<int>(1);
 
-    CUSOLVER_CHECK(getrf_func<T>()(cusolverDnHandle(),
+    CUSOLVER_CHECK(getrf_func<T>()(solverDnHandle(),
                                    M, N,
                                    in.get(), in.strides()[1],
                                    workspace,
diff --git a/src/backend/cuda/platform.cpp b/src/backend/cuda/platform.cpp
index fbfee69d34..48408ef105 100644
--- a/src/backend/cuda/platform.cpp
+++ b/src/backend/cuda/platform.cpp
@@ -24,7 +24,6 @@
 #include <err_cuda.hpp>
 #include <util.hpp>
 #include <host_memory.hpp>
-#include <common/InteropManager.hpp>
 
 using namespace std;
 
@@ -407,43 +406,43 @@ MemoryManagerPinned& pinnedMemoryManager()
     return *(inst.pinnedMemManager.get());
 }
 
-GraphicsManager& interopManager()
+GraphicsResourceManager& interopManager()
 {
     DeviceManager& inst = DeviceManager::getInstance();
 
     int id = cuda::getActiveDeviceId();
 
     if (! inst.gfxManagers[id] )
-        inst.gfxManagers[id].reset(new GraphicsManager());
+        inst.gfxManagers[id].reset(new GraphicsResourceManager());
 
     return *(inst.gfxManagers[id].get());
 }
 
-FFTManager& cufftManager()
+PlanCache& fftManager()
 {
     return DeviceManager::getInstance().cufftManagers[getActiveDeviceId()];
 }
 
-BlasHandle cublasHandle()
+BlasHandle blasHandle()
 {
     DeviceManager& instance = DeviceManager::getInstance();
 
     int id = cuda::getActiveDeviceId();
 
     if (! instance.cublasHandles[id] )
-        instance.cublasHandles[id].reset(new BlasHandleWrapper());
+        instance.cublasHandles[id].reset(new cublasHandle());
 
     return instance.cublasHandles[id].get()->get();
 }
 
-SolveHandle cusolverDnHandle()
+SolveHandle solverDnHandle()
 {
     DeviceManager& instance = DeviceManager::getInstance();
 
     int id = cuda::getActiveDeviceId();
 
     if (! instance.cusolverHandles[id] )
-        instance.cusolverHandles[id].reset(new SolveHandleWrapper());
+        instance.cusolverHandles[id].reset(new cusolverDnHandle());
 
     // FIXME
     // This is not an ideal case. It's just a hack.
@@ -463,14 +462,14 @@ SolveHandle cusolverDnHandle()
     return instance.cusolverHandles[id].get()->get();
 }
 
-SparseHandle cusparseHandle()
+SparseHandle sparseHandle()
 {
     DeviceManager& instance = DeviceManager::getInstance();
 
     int id = cuda::getActiveDeviceId();
 
     if (! instance.cusparseHandles[id] )
-        instance.cusparseHandles[id].reset(new SparseHandleWrapper());
+        instance.cusparseHandles[id].reset(new cusparseHandle());
 
     return instance.cusparseHandles[id].get()->get();
 }
diff --git a/src/backend/cuda/platform.hpp b/src/backend/cuda/platform.hpp
index 8440d58b81..ba660cda62 100644
--- a/src/backend/cuda/platform.hpp
+++ b/src/backend/cuda/platform.hpp
@@ -78,17 +78,15 @@ MemoryManager& memoryManager();
 
 MemoryManagerPinned& pinnedMemoryManager();
 
-typedef common::InteropManager<GraphicsResourceManager, CGR_t> GraphicsManager;
-GraphicsManager& interopManager();
+GraphicsResourceManager& interopManager();
 
-typedef common::FFTPlanCache<PlanCache, PlanType> FFTManager;
-FFTManager& cufftManager();
+PlanCache& fftManager();
 
-BlasHandle cublasHandle();
+BlasHandle blasHandle();
 
-SolveHandle cusolverDnHandle();
+SolveHandle solverDnHandle();
 
-SparseHandle cusparseHandle();
+SparseHandle sparseHandle();
 //
 ///////////////////////// END Sub-Managers /////////////////////
 
@@ -105,15 +103,15 @@ class DeviceManager
 
         friend MemoryManagerPinned& pinnedMemoryManager();
 
-        friend GraphicsManager& interopManager();
+        friend GraphicsResourceManager& interopManager();
 
-        friend FFTManager& cufftManager();
+        friend PlanCache& fftManager();
 
-        friend BlasHandle cublasHandle();
+        friend BlasHandle blasHandle();
 
-        friend SolveHandle cusolverDnHandle();
+        friend SolveHandle solverDnHandle();
 
-        friend SparseHandle cusparseHandle();
+        friend SparseHandle sparseHandle();
 
         friend std::string getDeviceInfo(int device);
 
@@ -166,15 +164,15 @@ class DeviceManager
 
         std::unique_ptr<MemoryManagerPinned> pinnedMemManager;
 
-        std::unique_ptr<GraphicsManager> gfxManagers[MAX_DEVICES];
+        std::unique_ptr<GraphicsResourceManager> gfxManagers[MAX_DEVICES];
 
-        FFTManager cufftManagers[MAX_DEVICES];
+        PlanCache cufftManagers[MAX_DEVICES];
 
-        std::unique_ptr<BlasHandleWrapper> cublasHandles[MAX_DEVICES];
+        std::unique_ptr<cublasHandle> cublasHandles[MAX_DEVICES];
 
-        std::unique_ptr<SolveHandleWrapper> cusolverHandles[MAX_DEVICES];
+        std::unique_ptr<cusolverDnHandle> cusolverHandles[MAX_DEVICES];
 
-        std::unique_ptr<SparseHandleWrapper> cusparseHandles[MAX_DEVICES];
+        std::unique_ptr<cusparseHandle> cusparseHandles[MAX_DEVICES];
 };
 
 }
diff --git a/src/backend/cuda/plot.cpp b/src/backend/cuda/plot.cpp
index 3c67e7d36b..e7a31537ff 100644
--- a/src/backend/cuda/plot.cpp
+++ b/src/backend/cuda/plot.cpp
@@ -16,7 +16,7 @@
 #include <join.hpp>
 #include <reduce.hpp>
 #include <reorder.hpp>
-#include <common/InteropManager.hpp>
+#include <GraphicsResourceManager.hpp>
 
 using af::dim4;
 
@@ -30,7 +30,7 @@ void copy_plot(const Array<T> &P, forge::Plot* plot)
     if(DeviceManager::checkGraphicsInteropCapability()) {
         const T *d_P = P.get();
 
-        GraphicsManager& intrpMngr = interopManager();
+        GraphicsResourceManager& intrpMngr = interopManager();
 
         cudaGraphicsResource_t *resources = intrpMngr.getBufferResource(plot);
         // Map resource. Copy data to VBO. Unmap resource.
diff --git a/src/backend/cuda/qr.cu b/src/backend/cuda/qr.cu
index 7a73512ea9..b058453310 100644
--- a/src/backend/cuda/qr.cu
+++ b/src/backend/cuda/qr.cu
@@ -130,7 +130,7 @@ void qr(Array<T> &q, Array<T> &r, Array<T> &t, const Array<T> &in)
 
     int lwork = 0;
 
-    CUSOLVER_CHECK(geqrf_buf_func<T>()(cusolverDnHandle(),
+    CUSOLVER_CHECK(geqrf_buf_func<T>()(solverDnHandle(),
                                        M, N,
                                        in_copy.get(), in_copy.strides()[1],
                                        &lwork));
@@ -140,7 +140,7 @@ void qr(Array<T> &q, Array<T> &r, Array<T> &t, const Array<T> &in)
     t = createEmptyArray<T>(af::dim4(min(M, N), 1, 1, 1));
     int *info = memAlloc<int>(1);
 
-    CUSOLVER_CHECK(geqrf_func<T>()(cusolverDnHandle(),
+    CUSOLVER_CHECK(geqrf_func<T>()(solverDnHandle(),
                                    M, N,
                                    in_copy.get(), in_copy.strides()[1],
                                    t.get(),
@@ -157,7 +157,7 @@ void qr(Array<T> &q, Array<T> &r, Array<T> &t, const Array<T> &in)
     dim4 qdims(M, mn);
     q = identity<T>(qdims);
 
-    CUSOLVER_CHECK(mqr_func<T>()(cusolverDnHandle(),
+    CUSOLVER_CHECK(mqr_func<T>()(solverDnHandle(),
                                  CUBLAS_SIDE_LEFT, CUBLAS_OP_N,
                                  q.dims()[0],
                                  q.dims()[1],
@@ -185,7 +185,7 @@ Array<T> qr_inplace(Array<T> &in)
 
     int lwork = 0;
 
-    CUSOLVER_CHECK(geqrf_buf_func<T>()(cusolverDnHandle(),
+    CUSOLVER_CHECK(geqrf_buf_func<T>()(solverDnHandle(),
                                        M, N,
                                        in.get(), in.strides()[1],
                                        &lwork));
@@ -193,7 +193,7 @@ Array<T> qr_inplace(Array<T> &in)
     T *workspace = memAlloc<T>(lwork);
     int *info = memAlloc<int>(1);
 
-    CUSOLVER_CHECK(geqrf_func<T>()(cusolverDnHandle(),
+    CUSOLVER_CHECK(geqrf_func<T>()(solverDnHandle(),
                                    M, N,
                                    in.get(), in.strides()[1],
                                    t.get(),
diff --git a/src/backend/cuda/solve.cu b/src/backend/cuda/solve.cu
index c4984c4868..7f3e7901ac 100644
--- a/src/backend/cuda/solve.cu
+++ b/src/backend/cuda/solve.cu
@@ -174,7 +174,7 @@ Array<T> solveLU(const Array<T> &A, const Array<int> &pivot,
 
     int *info = memAlloc<int>(1);
 
-    CUSOLVER_CHECK(getrs_func<T>()(cusolverDnHandle(),
+    CUSOLVER_CHECK(getrs_func<T>()(solverDnHandle(),
                                    CUBLAS_OP_N,
                                    N, NRHS,
                                    A.get(), A.strides()[1],
@@ -199,7 +199,7 @@ Array<T> generalSolve(const Array<T> &a, const Array<T> &b)
 
     int *info = memAlloc<int>(1);
 
-    CUSOLVER_CHECK(getrs_func<T>()(cusolverDnHandle(),
+    CUSOLVER_CHECK(getrs_func<T>()(solverDnHandle(),
                                    CUBLAS_OP_N,
                                    N, K,
                                    A.get(), A.strides()[1],
@@ -242,7 +242,7 @@ Array<T> leastSquares(const Array<T> &a, const Array<T> &b)
         int lwork = 0;
 
         // Get workspace needed for QR
-        CUSOLVER_CHECK(geqrf_solve_buf_func<T>()(cusolverDnHandle(),
+        CUSOLVER_CHECK(geqrf_solve_buf_func<T>()(solverDnHandle(),
                                                  A.dims()[0], A.dims()[1],
                                                  A.get(), A.strides()[1],
                                                  &lwork));
@@ -252,7 +252,7 @@ Array<T> leastSquares(const Array<T> &a, const Array<T> &b)
         int *info = memAlloc<int>(1);
 
         // In place Perform in place QR
-        CUSOLVER_CHECK(geqrf_solve_func<T>()(cusolverDnHandle(),
+        CUSOLVER_CHECK(geqrf_solve_func<T>()(solverDnHandle(),
                                              A.dims()[0], A.dims()[1],
                                              A.get(), A.strides()[1],
                                              t.get(),
@@ -270,7 +270,7 @@ Array<T> leastSquares(const Array<T> &a, const Array<T> &b)
         B.resetDims(dim4(N, K));
 
         // matmul(Q, Bpad)
-        CUSOLVER_CHECK(mqr_solve_func<T>()(cusolverDnHandle(),
+        CUSOLVER_CHECK(mqr_solve_func<T>()(solverDnHandle(),
                                            CUBLAS_SIDE_LEFT, CUBLAS_OP_N,
                                            B.dims()[0],
                                            B.dims()[1],
@@ -300,7 +300,7 @@ Array<T> leastSquares(const Array<T> &a, const Array<T> &b)
         int lwork = 0;
 
         // Get workspace needed for QR
-        CUSOLVER_CHECK(geqrf_solve_buf_func<T>()(cusolverDnHandle(),
+        CUSOLVER_CHECK(geqrf_solve_buf_func<T>()(solverDnHandle(),
                                                  A.dims()[0], A.dims()[1],
                                                  A.get(), A.strides()[1],
                                                  &lwork));
@@ -310,7 +310,7 @@ Array<T> leastSquares(const Array<T> &a, const Array<T> &b)
         int *info = memAlloc<int>(1);
 
         // In place Perform in place QR
-        CUSOLVER_CHECK(geqrf_solve_func<T>()(cusolverDnHandle(),
+        CUSOLVER_CHECK(geqrf_solve_func<T>()(solverDnHandle(),
                                              A.dims()[0], A.dims()[1],
                                              A.get(), A.strides()[1],
                                              t.get(),
@@ -318,7 +318,7 @@ Array<T> leastSquares(const Array<T> &a, const Array<T> &b)
                                              info));
 
         // matmul(Q1, B)
-        CUSOLVER_CHECK(mqr_solve_func<T>()(cusolverDnHandle(),
+        CUSOLVER_CHECK(mqr_solve_func<T>()(solverDnHandle(),
                                            CUBLAS_SIDE_LEFT,
                                            trans<T>(),
                                            M, K, N,
diff --git a/src/backend/cuda/sparse.cu b/src/backend/cuda/sparse.cu
index 49eb442d99..0fe53c438d 100644
--- a/src/backend/cuda/sparse.cu
+++ b/src/backend/cuda/sparse.cu
@@ -261,7 +261,7 @@ SparseArray<T> sparseConvertDenseToStorage(const Array<T> &in)
 
     int nNZ = -1;
     CUSPARSE_CHECK(nnz_func<T>()(
-                        cusparseHandle(),
+                        sparseHandle(),
                         dir,
                         M, N,
                         descr,
@@ -282,7 +282,7 @@ SparseArray<T> sparseConvertDenseToStorage(const Array<T> &in)
 
     if(stype == AF_STORAGE_CSR)
         CUSPARSE_CHECK(dense2csr_func<T>()(
-                        cusparseHandle(),
+                        sparseHandle(),
                         M, N,
                         descr,
                         in.get(), in.strides()[1],
@@ -290,7 +290,7 @@ SparseArray<T> sparseConvertDenseToStorage(const Array<T> &in)
                         values.get(), rowIdx.get(), colIdx.get()));
     else
         CUSPARSE_CHECK(dense2csc_func<T>()(
-                        cusparseHandle(),
+                        sparseHandle(),
                         M, N,
                         descr,
                         in.get(), in.strides()[1],
@@ -336,7 +336,7 @@ Array<T> sparseConvertStorageToDense(const SparseArray<T> &in)
 
     if(stype == AF_STORAGE_CSR)
         CUSPARSE_CHECK(csr2dense_func<T>()(
-                        cusparseHandle(),
+                        sparseHandle(),
                         M, N,
                         descr,
                         in.getValues().get(),
@@ -345,7 +345,7 @@ Array<T> sparseConvertStorageToDense(const SparseArray<T> &in)
                         dense.get(), d_strides1));
     else
         CUSPARSE_CHECK(csc2dense_func<T>()(
-                        cusparseHandle(),
+                        sparseHandle(),
                         M, N,
                         descr,
                         in.getValues().get(),
@@ -377,7 +377,7 @@ SparseArray<T> sparseConvertStorageToStorage(const SparseArray<T> &in)
 
         // cusparse function to expand compressed row into coordinate
         CUSPARSE_CHECK(cusparseXcsr2coo(
-                        cusparseHandle(),
+                        sparseHandle(),
                         in.getRowIdx().get(),
                         nNZ, in.dims()[0],
                         converted.getRowIdx().get(),
@@ -386,23 +386,23 @@ SparseArray<T> sparseConvertStorageToStorage(const SparseArray<T> &in)
         // Call sort
         size_t pBufferSizeInBytes = 0;
         CUSPARSE_CHECK(cusparseXcoosort_bufferSizeExt(
-                        cusparseHandle(),
+                        sparseHandle(),
                         in.dims()[0], in.dims()[1], nNZ,
                         converted.getRowIdx().get(), converted.getColIdx().get(),
                         &pBufferSizeInBytes));
         shared_ptr<char> pBuffer(memAlloc<char>(pBufferSizeInBytes), memFree<char>);
 
         shared_ptr<int> P(memAlloc<int>(nNZ), memFree<int>);
-        CUSPARSE_CHECK(cusparseCreateIdentityPermutation(cusparseHandle(), nNZ, P.get()));
+        CUSPARSE_CHECK(cusparseCreateIdentityPermutation(sparseHandle(), nNZ, P.get()));
 
         CUSPARSE_CHECK(cusparseXcoosortByColumn(
-                       cusparseHandle(),
+                       sparseHandle(),
                        in.dims()[0], in.dims()[1], nNZ,
                        converted.getRowIdx().get(), converted.getColIdx().get(),
                        P.get(), (void*)pBuffer.get()));
 
         CUSPARSE_CHECK(gthr_func<T>()(
-                       cusparseHandle(), nNZ,
+                       sparseHandle(), nNZ,
                        in.getValues().get(),
                        converted.getValues().get(),
                        P.get(), CUSPARSE_INDEX_BASE_ZERO));
@@ -421,23 +421,23 @@ SparseArray<T> sparseConvertStorageToStorage(const SparseArray<T> &in)
         {
             size_t pBufferSizeInBytes = 0;
             CUSPARSE_CHECK(cusparseXcoosort_bufferSizeExt(
-                            cusparseHandle(),
+                            sparseHandle(),
                             cooT.dims()[0], cooT.dims()[1], nNZ,
                             cooT.getRowIdx().get(), cooT.getColIdx().get(),
                             &pBufferSizeInBytes));
             shared_ptr<char> pBuffer(memAlloc<char>(pBufferSizeInBytes), memFree<char>);
 
             shared_ptr<int> P(memAlloc<int>(nNZ), memFree<int>);
-            CUSPARSE_CHECK(cusparseCreateIdentityPermutation(cusparseHandle(), nNZ, P.get()));
+            CUSPARSE_CHECK(cusparseCreateIdentityPermutation(sparseHandle(), nNZ, P.get()));
 
             CUSPARSE_CHECK(cusparseXcoosortByRow(
-                           cusparseHandle(),
+                           sparseHandle(),
                            cooT.dims()[0], cooT.dims()[1], nNZ,
                            cooT.getRowIdx().get(), cooT.getColIdx().get(),
                            P.get(), (void*)pBuffer.get()));
 
             CUSPARSE_CHECK(gthr_func<T>()(
-                           cusparseHandle(), nNZ,
+                           sparseHandle(), nNZ,
                            in.getValues().get(),
                            cooT.getValues().get(),
                            P.get(), CUSPARSE_INDEX_BASE_ZERO));
@@ -456,7 +456,7 @@ SparseArray<T> sparseConvertStorageToStorage(const SparseArray<T> &in)
 
         // cusparse function to compress row from coordinate
         CUSPARSE_CHECK(cusparseXcoo2csr(
-                        cusparseHandle(),
+                        sparseHandle(),
                         cooT.getRowIdx().get(),
                         nNZ, cooT.dims()[0],
                         converted.getRowIdx().get(),
diff --git a/src/backend/cuda/sparse_blas.cpp b/src/backend/cuda/sparse_blas.cpp
index be768d054d..df57c9c667 100644
--- a/src/backend/cuda/sparse_blas.cpp
+++ b/src/backend/cuda/sparse_blas.cpp
@@ -155,7 +155,7 @@ Array<T> matmul(const common::SparseArray<T> lhs, const Array<T> rhs,
     // and not OP(A) (gemm wants row/col of OP(A)).
     if(rDims[rColDim] == 1) {
         CUSPARSE_CHECK(csrmv_func<T>()(
-                       cusparseHandle(),
+                       sparseHandle(),
                        lOpts,
                        lDims[0], lDims[1], lhs.getNNZ(),
                        &alpha,
@@ -166,7 +166,7 @@ Array<T> matmul(const common::SparseArray<T> lhs, const Array<T> rhs,
                        out.get()));
     } else {
         CUSPARSE_CHECK(csrmm_func<T>()(
-                       cusparseHandle(),
+                       sparseHandle(),
                        lOpts,
                        lDims[0], rDims[rColDim], lDims[1], lhs.getNNZ(),
                        &alpha,
diff --git a/src/backend/cuda/surface.cpp b/src/backend/cuda/surface.cpp
index 95ff72645f..650968a61f 100644
--- a/src/backend/cuda/surface.cpp
+++ b/src/backend/cuda/surface.cpp
@@ -16,7 +16,7 @@
 #include <join.hpp>
 #include <reduce.hpp>
 #include <reorder.hpp>
-#include <common/InteropManager.hpp>
+#include <GraphicsResourceManager.hpp>
 
 using af::dim4;
 
@@ -30,7 +30,7 @@ void copy_surface(const Array<T> &P, forge::Surface* surface)
     if(DeviceManager::checkGraphicsInteropCapability()) {
         const T *d_P = P.get();
 
-        GraphicsManager& intrpMngr = interopManager();
+        GraphicsResourceManager& intrpMngr = interopManager();
 
         cudaGraphicsResource_t *resources = intrpMngr.getBufferResource(surface);
         // Map resource. Copy data to VBO. Unmap resource.
diff --git a/src/backend/cuda/svd.cu b/src/backend/cuda/svd.cu
index 9645aecf66..c323ab62ea 100644
--- a/src/backend/cuda/svd.cu
+++ b/src/backend/cuda/svd.cu
@@ -86,14 +86,14 @@ SVD_SPECIALIZE(cdouble, double, Z);
 
         int lwork = 0;
 
-        CUSOLVER_CHECK(gesvd_buf_func<T>(cusolverDnHandle(), M, N, &lwork));
+        CUSOLVER_CHECK(gesvd_buf_func<T>(solverDnHandle(), M, N, &lwork));
 
         T  *lWorkspace = memAlloc<T >(lwork);
         Tr *rWorkspace = memAlloc<Tr>(5 * std::min(M, N));
 
         int *info = memAlloc<int>(1);
 
-        gesvd_func<T, Tr>(cusolverDnHandle(), 'A', 'A', M, N, in.get(),
+        gesvd_func<T, Tr>(solverDnHandle(), 'A', 'A', M, N, in.get(),
                           M, s.get(), u.get(), M, vt.get(), N,
                           lWorkspace, lwork, rWorkspace, info);
 
diff --git a/src/backend/cuda/vector_field.cpp b/src/backend/cuda/vector_field.cpp
index d6eb919422..c875130a8b 100644
--- a/src/backend/cuda/vector_field.cpp
+++ b/src/backend/cuda/vector_field.cpp
@@ -13,7 +13,7 @@
 #include <vector_field.hpp>
 #include <err_cuda.hpp>
 #include <debug_cuda.hpp>
-#include <common/InteropManager.hpp>
+#include <GraphicsResourceManager.hpp>
 
 using af::dim4;
 
@@ -26,7 +26,7 @@ void copy_vector_field(const Array<T> &points, const Array<T> &directions,
                        forge::VectorField* vector_field)
 {
     if(DeviceManager::checkGraphicsInteropCapability()) {
-        GraphicsManager& intrpMngr = interopManager();
+        GraphicsResourceManager& intrpMngr = interopManager();
 
         cudaGraphicsResource_t *resources = intrpMngr.getBufferResource(vector_field);
 
diff --git a/src/backend/opencl/clfft.cpp b/src/backend/opencl/clfft.cpp
index dae1558fbe..483d88cfd6 100644
--- a/src/backend/opencl/clfft.cpp
+++ b/src/backend/opencl/clfft.cpp
@@ -122,7 +122,7 @@ PlanType findPlan(clfftLayout iLayout, clfftLayout oLayout,
     sprintf(key_str_temp, "%d:" SIZE_T_FRMT_SPECIFIER, (int)precision, batch);
     key_string.append(std::string(key_str_temp));
 
-    FFTManager &planner = opencl::clfftManager();
+    PlanCache &planner = opencl::fftManager();
 
     int planIndex = planner.find(key_string);
 
diff --git a/src/backend/opencl/fft.cpp b/src/backend/opencl/fft.cpp
index f3423ec6c0..56a1927d84 100644
--- a/src/backend/opencl/fft.cpp
+++ b/src/backend/opencl/fft.cpp
@@ -24,7 +24,7 @@ namespace opencl
 
 void setFFTPlanCacheSize(size_t numPlans)
 {
-    clfftManager().maxCacheSize(numPlans);
+    fftManager().maxCacheSize(numPlans);
 }
 
 template<typename T> struct Precision;
diff --git a/src/backend/opencl/hist_graphics.cpp b/src/backend/opencl/hist_graphics.cpp
index 41d7e2734c..701c8c337a 100644
--- a/src/backend/opencl/hist_graphics.cpp
+++ b/src/backend/opencl/hist_graphics.cpp
@@ -13,7 +13,7 @@
 #include <debug_opencl.hpp>
 #include <err_opencl.hpp>
 #include <hist_graphics.hpp>
-#include <common/InteropManager.hpp>
+#include <GraphicsResourceManager.hpp>
 
 namespace opencl
 {
@@ -27,7 +27,7 @@ void copy_histogram(const Array<T> &data, const forge::Histogram* hist)
         const cl::Buffer *d_P = data.get();
         size_t bytes = hist->verticesSize();
 
-        GraphicsManager& intrpMngr = interopManager();
+        GraphicsResourceManager& intrpMngr = interopManager();
 
         cl::Buffer **resources = intrpMngr.getBufferResource(hist);
 
diff --git a/src/backend/opencl/image.cpp b/src/backend/opencl/image.cpp
index 8429e1c71d..4df7f5cde5 100644
--- a/src/backend/opencl/image.cpp
+++ b/src/backend/opencl/image.cpp
@@ -13,7 +13,7 @@
 #include <debug_opencl.hpp>
 #include <err_opencl.hpp>
 #include <image.hpp>
-#include <common/InteropManager.hpp>
+#include <GraphicsResourceManager.hpp>
 
 #include <stdexcept>
 #include <vector>
@@ -27,7 +27,7 @@ void copy_image(const Array<T> &in, const forge::Image* image)
 {
     if (isGLSharingSupported()) {
         CheckGL("Begin opencl resource copy");
-        GraphicsManager& intrpMngr = interopManager();
+        GraphicsResourceManager& intrpMngr = interopManager();
 
         cl::Buffer **resources = intrpMngr.getBufferResource(image);
         const cl::Buffer *d_X = in.get();
diff --git a/src/backend/opencl/platform.cpp b/src/backend/opencl/platform.cpp
index 295fcccad5..eb72bf5fc8 100644
--- a/src/backend/opencl/platform.cpp
+++ b/src/backend/opencl/platform.cpp
@@ -860,19 +860,19 @@ MemoryManagerPinned& pinnedMemoryManager()
     return *(inst.pinnedMemManager.get());
 }
 
-GraphicsManager& interopManager()
+GraphicsResourceManager& interopManager()
 {
     DeviceManager& inst = DeviceManager::getInstance();
 
     int id = getActiveDeviceId();
 
     if (! inst.gfxManagers[id] )
-        inst.gfxManagers[id].reset(new GraphicsManager());
+        inst.gfxManagers[id].reset(new GraphicsResourceManager());
 
     return *(inst.gfxManagers[id].get());
 }
 
-FFTManager& clfftManager()
+PlanCache& fftManager()
 {
     return DeviceManager::getInstance().clfftManagers[getActiveDeviceId()];
 }
diff --git a/src/backend/opencl/platform.hpp b/src/backend/opencl/platform.hpp
index 71c2ae0c56..55aff48aac 100644
--- a/src/backend/opencl/platform.hpp
+++ b/src/backend/opencl/platform.hpp
@@ -89,11 +89,9 @@ MemoryManager& memoryManager();
 
 MemoryManagerPinned& pinnedMemoryManager();
 
-typedef common::InteropManager<GraphicsResourceManager, CGR_t> GraphicsManager;
-GraphicsManager& interopManager();
+GraphicsResourceManager& interopManager();
 
-typedef common::FFTPlanCache<PlanCache, PlanType> FFTManager;
-FFTManager& clfftManager();
+PlanCache& fftManager();
 //
 ///////////////////////// END Sub-Managers /////////////////////
 
@@ -103,9 +101,9 @@ class DeviceManager
 
     friend MemoryManagerPinned& pinnedMemoryManager();
 
-    friend GraphicsManager& interopManager();
+    friend GraphicsResourceManager& interopManager();
 
-    friend FFTManager& clfftManager();
+    friend PlanCache& fftManager();
 
     friend std::string getDeviceInfo();
 
@@ -178,7 +176,7 @@ class DeviceManager
         std::unique_ptr<MemoryManager> memManager;
         std::unique_ptr<MemoryManagerPinned> pinnedMemManager;
 
-        std::unique_ptr<GraphicsManager> gfxManagers[MAX_DEVICES];
-        FFTManager clfftManagers[MAX_DEVICES];
+        std::unique_ptr<GraphicsResourceManager> gfxManagers[MAX_DEVICES];
+        PlanCache clfftManagers[MAX_DEVICES];
 };
 }
diff --git a/src/backend/opencl/plot.cpp b/src/backend/opencl/plot.cpp
index a8eb7694e5..cd8134fe4f 100644
--- a/src/backend/opencl/plot.cpp
+++ b/src/backend/opencl/plot.cpp
@@ -12,7 +12,7 @@
 #include <Array.hpp>
 #include <debug_opencl.hpp>
 #include <err_opencl.hpp>
-#include <common/InteropManager.hpp>
+#include <GraphicsResourceManager.hpp>
 #include <join.hpp>
 #include <plot.hpp>
 #include <reduce.hpp>
@@ -32,7 +32,7 @@ void copy_plot(const Array<T> &P, forge::Plot* plot)
         const cl::Buffer *d_P = P.get();
         size_t bytes = plot->verticesSize();
 
-        GraphicsManager& intrpMngr = interopManager();
+        GraphicsResourceManager& intrpMngr = interopManager();
 
         cl::Buffer **resources = intrpMngr.getBufferResource(plot);
 
diff --git a/src/backend/opencl/surface.cpp b/src/backend/opencl/surface.cpp
index 29158de24e..ad296dbb54 100644
--- a/src/backend/opencl/surface.cpp
+++ b/src/backend/opencl/surface.cpp
@@ -12,7 +12,7 @@
 #include <Array.hpp>
 #include <debug_opencl.hpp>
 #include <err_opencl.hpp>
-#include <common/InteropManager.hpp>
+#include <GraphicsResourceManager.hpp>
 #include <join.hpp>
 #include <reduce.hpp>
 #include <reorder.hpp>
@@ -32,7 +32,7 @@ void copy_surface(const Array<T> &P, forge::Surface* surface)
         const cl::Buffer *d_P = P.get();
         size_t bytes = surface->verticesSize();
 
-        GraphicsManager& intrpMngr = interopManager();
+        GraphicsResourceManager& intrpMngr = interopManager();
 
         cl::Buffer **resources = intrpMngr.getBufferResource(surface);
 
diff --git a/src/backend/opencl/vector_field.cpp b/src/backend/opencl/vector_field.cpp
index 9debfc9d6a..53ac16d88a 100644
--- a/src/backend/opencl/vector_field.cpp
+++ b/src/backend/opencl/vector_field.cpp
@@ -12,7 +12,7 @@
 #include <Array.hpp>
 #include <debug_opencl.hpp>
 #include <err_opencl.hpp>
-#include <common/InteropManager.hpp>
+#include <GraphicsResourceManager.hpp>
 #include <vector_field.hpp>
 
 using af::dim4;
@@ -32,7 +32,7 @@ void copy_vector_field(const Array<T> &points, const Array<T> &directions,
         size_t pBytes = vector_field->verticesSize();
         size_t dBytes = vector_field->directionsSize();
 
-        GraphicsManager& intrpMngr = interopManager();
+        GraphicsResourceManager& intrpMngr = interopManager();
 
         cl::Buffer **resources = intrpMngr.getBufferResource(vector_field);
 

From dc3e84e8871a14d5c7c7ed6079400c2c4d3e7d35 Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Thu, 19 Jan 2017 14:44:55 +0530
Subject: [PATCH 28/82] fix memory manager clean up from device manager

---
 src/backend/opencl/memory.cpp   | 3 +--
 src/backend/opencl/platform.cpp | 5 +++++
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/backend/opencl/memory.cpp b/src/backend/opencl/memory.cpp
index e032f7bc6e..097bf1aadd 100644
--- a/src/backend/opencl/memory.cpp
+++ b/src/backend/opencl/memory.cpp
@@ -194,11 +194,10 @@ MemoryManagerPinned::~MemoryManagerPinned()
     common::lock_guard_t lock(this->memory_mutex);
     for (int n = 0; n < getDeviceCount(); n++) {
         opencl::setDevice(n);
-        garbageCollect();
+        this->garbageCollect();
         auto currIterator = pinnedMaps[n].begin();
         auto endIterator  = pinnedMaps[n].end();
         while (currIterator != endIterator) {
-            delete currIterator->second;
             pinnedMaps[n].erase(currIterator++);
         }
     }
diff --git a/src/backend/opencl/platform.cpp b/src/backend/opencl/platform.cpp
index eb72bf5fc8..3352f9092a 100644
--- a/src/backend/opencl/platform.cpp
+++ b/src/backend/opencl/platform.cpp
@@ -90,6 +90,11 @@ DeviceManager& DeviceManager::getInstance()
 
 DeviceManager::~DeviceManager()
 {
+    for (int i=0; i<getDeviceCount(); ++i)
+        delete gfxManagers[i].release();
+    delete memManager.release();
+    delete pinnedMemManager.release();
+
     //TODO: FIXME:
     // OpenCL libs on Windows platforms
     // are crashing the application at program exit

From e51255c34421e4d7f0927d44f970aff1d82c7884 Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Thu, 19 Jan 2017 16:42:51 +0530
Subject: [PATCH 29/82] Delegate thrust malloc/free calls to ArrayFire memory
 manager

---
 src/backend/cuda/ThrustAllocator.cuh     | 44 ++++++++++++++++++++++++
 src/backend/cuda/debug_cuda.hpp          |  7 ++++
 src/backend/cuda/kernel/regions.hpp      |  3 +-
 src/backend/cuda/kernel/sift_nonfree.hpp | 10 +++---
 4 files changed, 57 insertions(+), 7 deletions(-)
 create mode 100644 src/backend/cuda/ThrustAllocator.cuh

diff --git a/src/backend/cuda/ThrustAllocator.cuh b/src/backend/cuda/ThrustAllocator.cuh
new file mode 100644
index 0000000000..40cb899e01
--- /dev/null
+++ b/src/backend/cuda/ThrustAllocator.cuh
@@ -0,0 +1,44 @@
+/*******************************************************
+ * Copyright (c) 2016, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <thrust/device_malloc_allocator.h>
+#include <thrust/device_vector.h>
+#include <memory.hpp>
+
+//Below Class definition is found at the following URL
+//http://stackoverflow.com/questions/9007343/mix-custom-memory-managment-and-thrust-in-cuda
+
+namespace cuda
+{
+template<typename T>
+struct ThrustAllocator : thrust::device_malloc_allocator<T>
+{
+    // shorthand for the name of the base class
+    typedef thrust::device_malloc_allocator<T> super_t;
+
+    // get access to some of the base class's typedefs
+    // note that because we inherited from device_malloc_allocator,
+    // pointer is actually thrust::device_ptr<T>
+    typedef typename super_t::pointer   pointer;
+
+    typedef typename super_t::size_type size_type;
+
+    pointer allocate(size_type elements)
+    {
+        return thrust::device_ptr<T>(memAlloc<T>(elements));// delegate to ArrayFire allocator
+    }
+
+    void deallocate(pointer p, size_type n)
+    {
+        memFree<T>(p.get());// delegate to ArrayFire allocator
+    }
+};
+}
diff --git a/src/backend/cuda/debug_cuda.hpp b/src/backend/cuda/debug_cuda.hpp
index f7c60d37b2..5cf036b503 100644
--- a/src/backend/cuda/debug_cuda.hpp
+++ b/src/backend/cuda/debug_cuda.hpp
@@ -12,6 +12,13 @@
 #include <err_cuda.hpp>
 #include <thrust/version.h>
 #include <thrust/system/cuda/detail/par.h>
+#include <ThrustAllocator.cuh>
+
+namespace cuda
+{
+template<typename T>
+using ThrustVector = thrust::device_vector<T, cuda::ThrustAllocator<T> >;
+}
 
 #define THRUST_STREAM thrust::cuda::par.on(cuda::getActiveStream())
 
diff --git a/src/backend/cuda/kernel/regions.hpp b/src/backend/cuda/kernel/regions.hpp
index d9ef4080a8..6014e971eb 100644
--- a/src/backend/cuda/kernel/regions.hpp
+++ b/src/backend/cuda/kernel/regions.hpp
@@ -13,7 +13,6 @@
 #include <debug_cuda.hpp>
 #include <stdio.h>
 #include <memory.hpp>
-
 #include <thrust/system/cuda/detail/par.h>
 #include <thrust/adjacent_difference.h>
 #include <thrust/binary_search.h>
@@ -447,7 +446,7 @@ void regions(cuda::Param<T> out, cuda::CParam<char> in, cudaTextureObject_t tex)
     // compute.
     int num_bins = wrapped_tmp[size - 1] + 1;
 
-    thrust::device_vector<T> labels(num_bins);
+    cuda::ThrustVector<T> labels(num_bins);
 
     // Find the end of each section of values
     thrust::counting_iterator<T> search_begin(0);
diff --git a/src/backend/cuda/kernel/sift_nonfree.hpp b/src/backend/cuda/kernel/sift_nonfree.hpp
index 5fa051b82d..bdf233da77 100644
--- a/src/backend/cuda/kernel/sift_nonfree.hpp
+++ b/src/backend/cuda/kernel/sift_nonfree.hpp
@@ -1283,10 +1283,10 @@ std::vector< Param<T> > buildDoGPyr(
 }
 
 template <typename T>
-void update_permutation(thrust::device_ptr<T>& keys, thrust::device_vector<int>& permutation)
+void update_permutation(thrust::device_ptr<T>& keys, cuda::ThrustVector<int>& permutation)
 {
     // temporary storage for keys
-    thrust::device_vector<T> temp(permutation.size());
+    cuda::ThrustVector<T> temp(permutation.size());
 
     // permute the keys with the current reordering
     THRUST_SELECT((thrust::gather), permutation.begin(), permutation.end(), keys, temp.begin());
@@ -1296,10 +1296,10 @@ void update_permutation(thrust::device_ptr<T>& keys, thrust::device_vector<int>&
 }
 
 template <typename T>
-void apply_permutation(thrust::device_ptr<T>& keys, thrust::device_vector<int>& permutation)
+void apply_permutation(thrust::device_ptr<T>& keys, cuda::ThrustVector<int>& permutation)
 {
     // copy keys to temporary vector
-    thrust::device_vector<T> temp(keys, keys+permutation.size());
+    cuda::ThrustVector<T> temp(keys, keys+permutation.size());
 
     // permute the keys
     THRUST_SELECT((thrust::gather), permutation.begin(), permutation.end(), temp.begin(), keys);
@@ -1443,7 +1443,7 @@ void sift(unsigned* out_feat,
         thrust::device_ptr<float> interp_response_ptr = thrust::device_pointer_cast(d_interp_response);
         thrust::device_ptr<float> interp_size_ptr = thrust::device_pointer_cast(d_interp_size);
 
-        thrust::device_vector<int> permutation(interp_feat);
+        cuda::ThrustVector<int> permutation(interp_feat);
         thrust::sequence(permutation.begin(), permutation.end());
 
         update_permutation<float>(interp_size_ptr, permutation);

From 19953ffd894590e0fefa536651717ec849442a5f Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Fri, 20 Jan 2017 01:47:46 +0530
Subject: [PATCH 30/82] Make resource managers(cuda/opencl) access thread safe

---
 src/backend/common/FFTPlanCache.hpp   |   6 +-
 src/backend/common/InteropManager.hpp |   9 +-
 src/backend/common/types.hpp          |  17 +
 src/backend/cpu/types.hpp             |  17 +-
 src/backend/cuda/platform.cpp         |  37 +-
 src/backend/cuda/platform.hpp         |   4 +-
 src/backend/cuda/types.hpp            |  22 +-
 src/backend/opencl/platform.cpp       | 605 +++++++++++++-------------
 src/backend/opencl/platform.hpp       |   3 +
 src/backend/opencl/types.hpp          | 140 +++---
 10 files changed, 451 insertions(+), 409 deletions(-)
 create mode 100644 src/backend/common/types.hpp

diff --git a/src/backend/common/FFTPlanCache.hpp b/src/backend/common/FFTPlanCache.hpp
index bb0fa73803..6d0699c5c1 100644
--- a/src/backend/common/FFTPlanCache.hpp
+++ b/src/backend/common/FFTPlanCache.hpp
@@ -12,6 +12,7 @@
 #include <deque>
 #include <string>
 #include <utility>
+#include <common/types.hpp>
 
 namespace common
 {
@@ -34,6 +35,7 @@ class FFTPlanCache
         }
 
         inline void maxCacheSize(size_t size) {
+            lock_guard_t lock(mutex);
             mCache.resize(size, std::make_pair<std::string, P>(std::string(""), 0));
         }
 
@@ -63,6 +65,7 @@ class FFTPlanCache
         // pops plan from the back of cache(queue)
         void pop() {
             if (!mCache.empty()) {
+                lock_guard_t lock(mutex);
                 // destroy the cufft plan associated with the
                 // least recently used plan
                 static_cast<T*>(this)->removePlan(mCache.back().second);
@@ -73,6 +76,7 @@ class FFTPlanCache
 
         // pushes plan to the front of cache(queue)
         void push(std::string key, P plan) {
+            lock_guard_t lock(mutex);
             if (mCache.size()>mMaxCacheSize) {
                 pop();
             }
@@ -86,6 +90,6 @@ class FFTPlanCache
         size_t       mMaxCacheSize;
 
         std::deque< std::pair<std::string, P> > mCache;
-
+        mutex_t mutex;
 };
 }
diff --git a/src/backend/common/InteropManager.hpp b/src/backend/common/InteropManager.hpp
index f32a952e27..6d0d6f85aa 100644
--- a/src/backend/common/InteropManager.hpp
+++ b/src/backend/common/InteropManager.hpp
@@ -16,6 +16,7 @@
 #include <cstdio>
 #include <map>
 #include <vector>
+#include <common/types.hpp>
 
 namespace common
 {
@@ -38,6 +39,7 @@ class InteropManager
         }
 
         R* getBufferResource(const forge::Image* image) {
+            lock_guard_t lock(mutex);
             void * key = (void*)image;
 
             if (interopMap.find(key) == interopMap.end()) {
@@ -51,6 +53,7 @@ class InteropManager
         }
 
         R* getBufferResource(const forge::Plot* plot) {
+            lock_guard_t lock(mutex);
             void * key = (void*)plot;
 
             if (interopMap.find(key) == interopMap.end()) {
@@ -64,6 +67,7 @@ class InteropManager
         }
 
         R* getBufferResource(const forge::Histogram* histogram) {
+            lock_guard_t lock(mutex);
             void * key = (void*)histogram;
 
             if (interopMap.find(key) == interopMap.end()) {
@@ -77,6 +81,7 @@ class InteropManager
         }
 
         R* getBufferResource(const forge::Surface* surface) {
+            lock_guard_t lock(mutex);
             void * key = (void*)surface;
 
             if (interopMap.find(key) == interopMap.end()) {
@@ -90,6 +95,7 @@ class InteropManager
         }
 
         R* getBufferResource(const forge::VectorField* field) {
+            lock_guard_t lock(mutex);
             void * key = (void*)field;
 
             if (interopMap.find(key) == interopMap.end()) {
@@ -116,8 +122,9 @@ class InteropManager
             }
         }
 
-    private:
+        //Attributes
         std::map<void *, std::vector<R> > interopMap;
+        mutex_t mutex;
 };
 }
 #endif
diff --git a/src/backend/common/types.hpp b/src/backend/common/types.hpp
new file mode 100644
index 0000000000..9d08b04d48
--- /dev/null
+++ b/src/backend/common/types.hpp
@@ -0,0 +1,17 @@
+/*******************************************************
+ * Copyright (c) 2014, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+#include <mutex>
+
+namespace common
+{
+typedef std::recursive_mutex mutex_t;
+typedef std::lock_guard<mutex_t> lock_guard_t;
+}
diff --git a/src/backend/cpu/types.hpp b/src/backend/cpu/types.hpp
index 0776df783c..84e62f8286 100644
--- a/src/backend/cpu/types.hpp
+++ b/src/backend/cpu/types.hpp
@@ -12,14 +12,13 @@
 
 namespace cpu
 {
-    typedef std::complex<float>     cfloat;
-    typedef std::complex<double>    cdouble;
-    typedef unsigned int            uint;
-    typedef unsigned char           uchar;
-    typedef unsigned short          ushort;
-
-    template<typename T> struct is_complex          { static const bool value = false;  };
-    template<> struct           is_complex<cfloat>  { static const bool value = true;   };
-    template<> struct           is_complex<cdouble> { static const bool value = true;   };
+typedef std::complex<float>     cfloat;
+typedef std::complex<double>    cdouble;
+typedef unsigned int   uint;
+typedef unsigned char  uchar;
+typedef unsigned short ushort;
 
+template<typename T> struct is_complex          { static const bool value = false;  };
+template<> struct           is_complex<cfloat>  { static const bool value = true;   };
+template<> struct           is_complex<cdouble> { static const bool value = true;   };
 }
diff --git a/src/backend/cuda/platform.cpp b/src/backend/cuda/platform.cpp
index 48408ef105..0c19208311 100644
--- a/src/backend/cuda/platform.cpp
+++ b/src/backend/cuda/platform.cpp
@@ -408,12 +408,13 @@ MemoryManagerPinned& pinnedMemoryManager()
 
 GraphicsResourceManager& interopManager()
 {
-    DeviceManager& inst = DeviceManager::getInstance();
+    static std::once_flag initFlags[DeviceManager::MAX_DEVICES];
 
-    int id = cuda::getActiveDeviceId();
+    int id = getActiveDeviceId();
+
+    DeviceManager& inst = DeviceManager::getInstance();
 
-    if (! inst.gfxManagers[id] )
-        inst.gfxManagers[id].reset(new GraphicsResourceManager());
+    std::call_once(initFlags[id], [&]{ inst.gfxManagers[id].reset(new GraphicsResourceManager()); });
 
     return *(inst.gfxManagers[id].get());
 }
@@ -425,24 +426,26 @@ PlanCache& fftManager()
 
 BlasHandle blasHandle()
 {
-    DeviceManager& instance = DeviceManager::getInstance();
+    static std::once_flag initFlags[DeviceManager::MAX_DEVICES];
 
     int id = cuda::getActiveDeviceId();
 
-    if (! instance.cublasHandles[id] )
-        instance.cublasHandles[id].reset(new cublasHandle());
+    DeviceManager& inst = DeviceManager::getInstance();
+
+    std::call_once(initFlags[id], [&]{ inst.cublasHandles[id].reset(new cublasHandle()); });
 
-    return instance.cublasHandles[id].get()->get();
+    return inst.cublasHandles[id].get()->get();
 }
 
 SolveHandle solverDnHandle()
 {
-    DeviceManager& instance = DeviceManager::getInstance();
+    static std::once_flag initFlags[DeviceManager::MAX_DEVICES];
 
     int id = cuda::getActiveDeviceId();
 
-    if (! instance.cusolverHandles[id] )
-        instance.cusolverHandles[id].reset(new cusolverDnHandle());
+    DeviceManager& inst = DeviceManager::getInstance();
+
+    std::call_once(initFlags[id], [&]{ inst.cusolverHandles[id].reset(new cusolverDnHandle()); });
 
     // FIXME
     // This is not an ideal case. It's just a hack.
@@ -459,17 +462,18 @@ SolveHandle solverDnHandle()
     //
     CUDA_CHECK(cudaStreamSynchronize(cuda::getStream(id)));
 
-    return instance.cusolverHandles[id].get()->get();
+    return inst.cusolverHandles[id].get()->get();
 }
 
 SparseHandle sparseHandle()
 {
-    DeviceManager& instance = DeviceManager::getInstance();
+    static std::once_flag initFlags[DeviceManager::MAX_DEVICES];
 
     int id = cuda::getActiveDeviceId();
 
-    if (! instance.cusparseHandles[id] )
-        instance.cusparseHandles[id].reset(new cusparseHandle());
+    DeviceManager& inst = DeviceManager::getInstance();
+
+    std::call_once(initFlags[id], [&]{ inst.cusparseHandles[id].reset(new cusparseHandle()); });
 
     return instance.cusparseHandles[id].get()->get();
 }
@@ -517,6 +521,7 @@ DeviceManager::DeviceManager()
 
 void DeviceManager::sortDevices(sort_mode mode)
 {
+    lock_guard_t lock(deviceMutex);
     switch(mode) {
         case memory :
             std::stable_sort(cuDevices.begin(), cuDevices.end(), card_compare_mem);
@@ -537,6 +542,8 @@ int DeviceManager::setActiveDevice(int device, int nId)
 {
     static bool first = true;
 
+    lock_guard_t lock(deviceMutex);
+
     int numDevices = cuDevices.size();
 
     if(device > numDevices) return -1;
diff --git a/src/backend/cuda/platform.hpp b/src/backend/cuda/platform.hpp
index ba660cda62..ff2a9f8a79 100644
--- a/src/backend/cuda/platform.hpp
+++ b/src/backend/cuda/platform.hpp
@@ -20,10 +20,10 @@
 #include <cublas.hpp>
 #include <cusolverDn.hpp>
 #include <cusparse.hpp>
+#include <common/types.hpp>
 
 namespace cuda
 {
-
 int getBackend();
 
 std::string getDeviceInfo();
@@ -148,6 +148,7 @@ class DeviceManager
         void operator=(DeviceManager const&);
 
         // Attributes
+        common::mutex_t deviceMutex;
         std::vector<cudaDevice_t> cuDevices;
 
         enum sort_mode {flops = 0, memory = 1, compute = 2, none = 3};
@@ -174,5 +175,4 @@ class DeviceManager
 
         std::unique_ptr<cusparseHandle> cusparseHandles[MAX_DEVICES];
 };
-
 }
diff --git a/src/backend/cuda/types.hpp b/src/backend/cuda/types.hpp
index 26d0bb658d..08aab5f374 100644
--- a/src/backend/cuda/types.hpp
+++ b/src/backend/cuda/types.hpp
@@ -13,17 +13,17 @@
 
 namespace cuda
 {
-    typedef cuFloatComplex   cfloat;
-    typedef cuDoubleComplex cdouble;
-    typedef unsigned int   uint;
-    typedef unsigned char  uchar;
-    typedef unsigned short ushort;
+typedef cuFloatComplex   cfloat;
+typedef cuDoubleComplex cdouble;
+typedef unsigned int   uint;
+typedef unsigned char  uchar;
+typedef unsigned short ushort;
 
-    template<typename T> struct is_complex          { static const bool value = false;  };
-    template<> struct           is_complex<cfloat>  { static const bool value = true;   };
-    template<> struct           is_complex<cdouble> { static const bool value = true;   };
+template<typename T> struct is_complex          { static const bool value = false;  };
+template<> struct           is_complex<cfloat>  { static const bool value = true;   };
+template<> struct           is_complex<cdouble> { static const bool value = true;   };
 
-    template<typename T, bool binary> const std::string cuMangledName(const char *fn);
-    template<typename T > const char *afShortName(bool caps = true);
-    template<typename T > const char *irname();
+template<typename T, bool binary> const std::string cuMangledName(const char *fn);
+template<typename T > const char *afShortName(bool caps = true);
+template<typename T > const char *irname();
 }
diff --git a/src/backend/opencl/platform.cpp b/src/backend/opencl/platform.cpp
index 3352f9092a..910d500a58 100644
--- a/src/backend/opencl/platform.cpp
+++ b/src/backend/opencl/platform.cpp
@@ -56,7 +56,6 @@ using cl::Device;
 
 namespace opencl
 {
-
 #if defined (OS_MAC)
 static const std::string CL_GL_SHARING_EXT = "cl_APPLE_gl_sharing";
 #else
@@ -82,39 +81,6 @@ int getBackend()
     return AF_BACKEND_OPENCL;
 }
 
-DeviceManager& DeviceManager::getInstance()
-{
-    static DeviceManager my_instance;
-    return my_instance;
-}
-
-DeviceManager::~DeviceManager()
-{
-    for (int i=0; i<getDeviceCount(); ++i)
-        delete gfxManagers[i].release();
-    delete memManager.release();
-    delete pinnedMemManager.release();
-
-    //TODO: FIXME:
-    // OpenCL libs on Windows platforms
-    // are crashing the application at program exit
-    // most probably a reference counting issue based
-    // on the investigation done so far. This problem
-    // doesn't seem to happen on Linux or MacOSX.
-    // So, clean up OpenCL resources on non-Windows platforms
-#ifndef OS_WIN
-    for (auto q: mQueues) delete q;
-    for (auto d : mDevices) delete d;
-    for (auto c : mContexts) delete c;
-#endif
-}
-
-void DeviceManager::setContext(int device)
-{
-    mActiveQId = device;
-    mActiveCtxId = device;
-}
-
 static inline bool verify_present(std::string pname, const char *ref)
 {
     return pname.find(ref) != std::string::npos;
@@ -208,7 +174,6 @@ static afcl::deviceType getDeviceTypeEnum(cl::Device dev)
     return (afcl::deviceType)dev.getInfo<CL_DEVICE_TYPE>();
 }
 
-
 static afcl::platform getPlatformEnum(cl::Device dev)
 {
     std::string pname = getPlatformName(dev);
@@ -221,132 +186,6 @@ static afcl::platform getPlatformEnum(cl::Device dev)
     return AFCL_PLATFORM_UNKNOWN;
 }
 
-
-DeviceManager::DeviceManager()
-    : mUserDeviceOffset(0), mActiveCtxId(0), mActiveQId(0)
-{
-    std::vector<cl::Platform>   platforms;
-    Platform::get(&platforms);
-
-    // This is all we need because the sort takes care of the order of devices
-#ifdef OS_MAC
-    cl_device_type DEVICE_TYPES = CL_DEVICE_TYPE_GPU;
-#else
-    cl_device_type DEVICE_TYPES = CL_DEVICE_TYPE_ALL;
-#endif
-
-    std::string deviceENV = getEnvVar("AF_OPENCL_DEVICE_TYPE");
-
-    if (deviceENV.compare("GPU") == 0) {
-        DEVICE_TYPES = CL_DEVICE_TYPE_GPU;
-    } else if (deviceENV.compare("CPU") == 0) {
-        DEVICE_TYPES = CL_DEVICE_TYPE_CPU;
-    } else if (deviceENV.compare("ACC") >= 0) {
-        DEVICE_TYPES = CL_DEVICE_TYPE_ACCELERATOR;
-    }
-
-
-
-    // Iterate through platforms, get all available devices and store them
-    for (auto &platform : platforms) {
-        std::vector<Device> current_devices;
-
-        try {
-            platform.getDevices(DEVICE_TYPES, &current_devices);
-        } catch(const cl::Error &err) {
-            if (err.err() != CL_DEVICE_NOT_FOUND) {
-                throw;
-            }
-        }
-
-        for (auto dev : current_devices) {
-            mDevices.push_back(new Device(dev));
-        }
-    }
-
-    int nDevices = mDevices.size();
-
-    if (nDevices == 0) AF_ERROR("No OpenCL devices found", AF_ERR_RUNTIME);
-
-    // Sort OpenCL devices based on default criteria
-    std::stable_sort(mDevices.begin(), mDevices.end(), compare_default);
-
-    // Create contexts and queues once the sort is done
-    for (int i = 0; i < nDevices; i++) {
-        cl_platform_id device_platform = mDevices[i]->getInfo<CL_DEVICE_PLATFORM>();
-        cl_context_properties cps[3] = {CL_CONTEXT_PLATFORM,
-                                        (cl_context_properties)(device_platform),
-                                        0};
-
-        Context *ctx = new Context(*mDevices[i], cps);
-        CommandQueue *cq = new CommandQueue(*ctx, *mDevices[i]);
-        mContexts.push_back(ctx);
-        mQueues.push_back(cq);
-        mIsGLSharingOn.push_back(false);
-        mDeviceTypes.push_back(getDeviceTypeEnum(*mDevices[i]));
-        mPlatforms.push_back(getPlatformEnum(*mDevices[i]));
-    }
-
-    bool default_device_set = false;
-    deviceENV = getEnvVar("AF_OPENCL_DEFAULT_DEVICE");
-    if(!deviceENV.empty()) {
-        std::stringstream s(deviceENV);
-        int def_device = -1;
-        s >> def_device;
-        if(def_device < 0 || def_device >= (int)nDevices) {
-            printf("WARNING: AF_OPENCL_DEFAULT_DEVICE is out of range\n");
-            printf("Setting default device as 0\n");
-        } else {
-            setContext(def_device);
-            default_device_set = true;
-        }
-    }
-
-    deviceENV = getEnvVar("AF_OPENCL_DEFAULT_DEVICE_TYPE");
-    if (!default_device_set && !deviceENV.empty())
-    {
-        cl_device_type default_device_type = CL_DEVICE_TYPE_GPU;
-        if (deviceENV.compare("CPU") == 0) {
-            default_device_type = CL_DEVICE_TYPE_CPU;
-        } else if (deviceENV.compare("ACC") >= 0) {
-            default_device_type = CL_DEVICE_TYPE_ACCELERATOR;
-        }
-
-        bool default_device_set = false;
-        for (int i = 0; i < nDevices; i++) {
-            if (mDevices[i]->getInfo<CL_DEVICE_TYPE>() == default_device_type) {
-                default_device_set = true;
-                setContext(i);
-                break;
-            }
-        }
-
-        if (!default_device_set) {
-            printf("WARNING: AF_OPENCL_DEFAULT_DEVICE_TYPE=%s is not available\n",
-                    deviceENV.c_str());
-            printf("Using default device as 0\n");
-        }
-    }
-
-#if defined(WITH_GRAPHICS)
-    // Define AF_DISABLE_GRAPHICS with any value to disable initialization
-    std::string noGraphicsENV = getEnvVar("AF_DISABLE_GRAPHICS");
-    if(noGraphicsENV.empty()) { // If AF_DISABLE_GRAPHICS is not defined
-        try {
-            /* loop over devices and replace contexts with
-             * OpenGL shared contexts whereever applicable */
-            int devCount = mDevices.size();
-            forge::Window* wHandle = graphics::ForgeManager::getInstance().getMainWindow();
-            for(int i=0; i<devCount; ++i)
-                markDeviceForInterop(i, wHandle);
-        } catch (...) {
-        }
-    }
-#endif
-    mUserDeviceOffset = mDevices.size();
-}
-
-
 // http://stackoverflow.com/questions/216823/whats-the-best-way-to-trim-stdstring/217605#217605
 // trim from start
 static inline std::string &ltrim(std::string &s)
@@ -627,150 +466,56 @@ bool checkExtnAvailability(const Device &pDevice, std::string pName)
     return ret_val;
 }
 
-#if defined(WITH_GRAPHICS)
-void DeviceManager::markDeviceForInterop(const int device, const forge::Window* wHandle)
+void addDeviceContext(cl_device_id dev, cl_context ctx, cl_command_queue que)
 {
-    try {
-        if (device >= (int)mQueues.size() ||
-                device>= (int)DeviceManager::MAX_DEVICES) {
-            throw cl::Error(CL_INVALID_DEVICE, "Invalid device passed for CL-GL Interop");
-        } else {
-            mQueues[device]->finish();
+    clRetainDevice(dev);
+    clRetainContext(ctx);
+    clRetainCommandQueue(que);
 
-            // check if the device has CL_GL sharing extension enabled
-            bool temp = checkExtnAvailability(*mDevices[device], CL_GL_SHARING_EXT);
-            if (!temp) {
-                /* return silently if given device has not OpenGL sharing extension
-                 * enabled so that regular queue is used for it */
-                return;
-            }
+    DeviceManager& devMngr   = DeviceManager::getInstance();
 
-            // call forge to get OpenGL sharing context and details
-            cl::Platform plat(mDevices[device]->getInfo<CL_DEVICE_PLATFORM>());
+    common::lock_guard_t lock(devMngr.deviceMutex);
 
-#ifdef OS_MAC
-            CGLContextObj cgl_current_ctx = CGLGetCurrentContext();
-            CGLShareGroupObj cgl_share_group = CGLGetShareGroup(cgl_current_ctx);
+    cl::Device* tDevice      = new cl::Device(dev);
+    cl::Context* tContext    = new cl::Context(ctx);
+    cl::CommandQueue* tQueue = (que==NULL ?
+            new cl::CommandQueue(*tContext, *tDevice) : new cl::CommandQueue(que));
+    devMngr.mDevices.push_back(tDevice);
+    devMngr.mContexts.push_back(tContext);
+    devMngr.mQueues.push_back(tQueue);
+    devMngr.mPlatforms.push_back(getPlatformEnum(*tDevice));
+    // FIXME: add OpenGL Interop for user provided contexts later
+    devMngr.mIsGLSharingOn.push_back(false);
+}
 
-            cl_context_properties cps[] = {
-                CL_CONTEXT_PROPERTY_USE_CGL_SHAREGROUP_APPLE, (cl_context_properties)cgl_share_group,
-                0
-            };
-#else
-            cl_context_properties cps[] = {
-                CL_GL_CONTEXT_KHR, (cl_context_properties)wHandle->context(),
-#if defined(_WIN32) || defined(_MSC_VER)
-                CL_WGL_HDC_KHR, (cl_context_properties)wHandle->display(),
-#else
-                CL_GLX_DISPLAY_KHR, (cl_context_properties)wHandle->display(),
-#endif
-                CL_CONTEXT_PLATFORM, (cl_context_properties)plat(),
-                0
-            };
+void setDeviceContext(cl_device_id dev, cl_context ctx)
+{
+    // FIXME: add OpenGL Interop for user provided contexts later
+    DeviceManager& devMngr = DeviceManager::getInstance();
 
-            // Check if current OpenCL device is belongs to the OpenGL context
-            {
-                cl_context_properties test_cps[] = {
-                    CL_GL_CONTEXT_KHR, (cl_context_properties)wHandle->context(),
-                    CL_CONTEXT_PLATFORM, (cl_context_properties)plat(),
-                    0
-                };
+    common::lock_guard_t lock(devMngr.deviceMutex);
 
-                // Load the extension
-                // If cl_khr_gl_sharing is available, this function should be present
-                // This has been checked earlier, it comes to this point only if it is found
-                auto func = (clGetGLContextInfoKHR_fn)
-                    clGetExtensionFunctionAddressForPlatform(plat(), "clGetGLContextInfoKHR");
+    const int dCount = devMngr.mDevices.size();
+    for (int i=0; i<dCount; ++i) {
+        if(devMngr.mDevices[i]->operator()()==dev &&
+                devMngr.mContexts[i]->operator()()==ctx) {
+            setDevice(i);
+            return;
+        }
+    }
+    AF_ERROR("No matching device found", AF_ERR_ARG);
+}
 
-                // If the function doesn't load, bail early
-                if (!func) return;
+void removeDeviceContext(cl_device_id dev, cl_context ctx)
+{
+    if (getDevice()() == dev && getContext()()==ctx) {
+        AF_ERROR("Cannot pop the device currently in use", AF_ERR_ARG);
+    }
 
-                // Get all devices associated with opengl context
-                std::vector<cl_device_id> devices(16);
-                size_t ret = 0;
-                cl_int err = func(test_cps,
-                                  CL_DEVICES_FOR_GL_CONTEXT_KHR,
-                                  devices.size() * sizeof(cl_device_id),
-                                  &devices[0],
-                                  &ret);
-                if (err != CL_SUCCESS) return;
-                int num = ret / sizeof(cl_device_id);
-                devices.resize(num);
-
-                // Check if current device is present in the associated devices
-                cl_device_id current_device = (*mDevices[device])();
-                auto res = std::find(std::begin(devices),
-                                     std::end(devices),
-                                     current_device);
-
-                if (res == std::end(devices)) return;
-            }
-#endif
-
-            // Change current device to use GL sharing
-            Context * ctx = new Context(*mDevices[device], cps);
-            CommandQueue * cq = new CommandQueue(*ctx, *mDevices[device]);
-
-            // May be fixes the AMD GL issues we see on windows?
-#if !defined(_WIN32) && !defined(_MSC_VER)
-            delete mContexts[device];
-            delete mQueues[device];
-#endif
-
-            mContexts[device] = ctx;
-            mQueues[device] = cq;
-            mIsGLSharingOn[device] = true;
-        }
-    } catch (const cl::Error &ex) {
-        /* If replacing the original context with GL shared context
-         * failes, don't throw an error and instead fall back to
-         * original context and use copy via host to support graphics
-         * on that particular OpenCL device. So mark it as no GL sharing */
-    }
-}
-#endif
-
-void addDeviceContext(cl_device_id dev, cl_context ctx, cl_command_queue que)
-{
-    clRetainDevice(dev);
-    clRetainContext(ctx);
-    clRetainCommandQueue(que);
-
-    DeviceManager& devMngr   = DeviceManager::getInstance();
-    cl::Device* tDevice      = new cl::Device(dev);
-    cl::Context* tContext    = new cl::Context(ctx);
-    cl::CommandQueue* tQueue = (que==NULL ?
-            new cl::CommandQueue(*tContext, *tDevice) : new cl::CommandQueue(que));
-    devMngr.mDevices.push_back(tDevice);
-    devMngr.mContexts.push_back(tContext);
-    devMngr.mQueues.push_back(tQueue);
-    devMngr.mPlatforms.push_back(getPlatformEnum(*tDevice));
-    // FIXME: add OpenGL Interop for user provided contexts later
-    devMngr.mIsGLSharingOn.push_back(false);
-}
-
-void setDeviceContext(cl_device_id dev, cl_context ctx)
-{
-    // FIXME: add OpenGL Interop for user provided contexts later
     DeviceManager& devMngr = DeviceManager::getInstance();
-    const int dCount = devMngr.mDevices.size();
-    for (int i=0; i<dCount; ++i) {
-        if(devMngr.mDevices[i]->operator()()==dev &&
-                devMngr.mContexts[i]->operator()()==ctx) {
-            setDevice(i);
-            return;
-        }
-    }
-    AF_ERROR("No matching device found", AF_ERR_ARG);
-}
 
-void removeDeviceContext(cl_device_id dev, cl_context ctx)
-{
-    if (getDevice()() == dev && getContext()()==ctx) {
-        AF_ERROR("Cannot pop the device currently in use", AF_ERR_ARG);
-    }
+    common::lock_guard_t lock(devMngr.deviceMutex);
 
-    DeviceManager& devMngr = DeviceManager::getInstance();
     const int dCount = devMngr.mDevices.size();
     int deleteIdx = -1;
     for (int i = 0; i<dCount; ++i) {
@@ -816,7 +561,6 @@ bool synchronize_calls() {
     return sync;
 }
 
-
 unsigned getMaxJitSize()
 {
 #if defined(OS_MAC)
@@ -849,7 +593,7 @@ MemoryManager& memoryManager()
 
     DeviceManager& inst = DeviceManager::getInstance();
 
-    std::call_once(flag, [&]() { inst.memManager.reset(new MemoryManager()); });
+    std::call_once(flag, [&]{ inst.memManager.reset(new MemoryManager()); });
 
     return *(inst.memManager.get());
 }
@@ -860,19 +604,20 @@ MemoryManagerPinned& pinnedMemoryManager()
 
     DeviceManager& inst = DeviceManager::getInstance();
 
-    std::call_once(flag, [&]() { inst.pinnedMemManager.reset(new MemoryManagerPinned()); });
+    std::call_once(flag, [&]{ inst.pinnedMemManager.reset(new MemoryManagerPinned()); });
 
     return *(inst.pinnedMemManager.get());
 }
 
 GraphicsResourceManager& interopManager()
 {
-    DeviceManager& inst = DeviceManager::getInstance();
+    static std::once_flag initFlags[DeviceManager::MAX_DEVICES];
 
     int id = getActiveDeviceId();
 
-    if (! inst.gfxManagers[id] )
-        inst.gfxManagers[id].reset(new GraphicsResourceManager());
+    DeviceManager& inst = DeviceManager::getInstance();
+
+    std::call_once(initFlags[id], [&]{ inst.gfxManagers[id].reset(new GraphicsResourceManager()); });
 
     return *(inst.gfxManagers[id].get());
 }
@@ -882,6 +627,268 @@ PlanCache& fftManager()
     return DeviceManager::getInstance().clfftManagers[getActiveDeviceId()];
 }
 
+DeviceManager& DeviceManager::getInstance()
+{
+    static DeviceManager my_instance;
+    return my_instance;
+}
+
+DeviceManager::~DeviceManager()
+{
+    for (int i=0; i<getDeviceCount(); ++i)
+        delete gfxManagers[i].release();
+    delete memManager.release();
+    delete pinnedMemManager.release();
+
+    //TODO: FIXME:
+    // OpenCL libs on Windows platforms
+    // are crashing the application at program exit
+    // most probably a reference counting issue based
+    // on the investigation done so far. This problem
+    // doesn't seem to happen on Linux or MacOSX.
+    // So, clean up OpenCL resources on non-Windows platforms
+#ifndef OS_WIN
+    for (auto q: mQueues) delete q;
+    for (auto d : mDevices) delete d;
+    for (auto c : mContexts) delete c;
+#endif
+}
+
+void DeviceManager::setContext(int device)
+{
+    common::lock_guard_t lock(deviceMutex);
+
+    mActiveQId = device;
+    mActiveCtxId = device;
+}
+
+DeviceManager::DeviceManager()
+    : mUserDeviceOffset(0), mActiveCtxId(0), mActiveQId(0)
+{
+    std::vector<cl::Platform>   platforms;
+    Platform::get(&platforms);
+
+    // This is all we need because the sort takes care of the order of devices
+#ifdef OS_MAC
+    cl_device_type DEVICE_TYPES = CL_DEVICE_TYPE_GPU;
+#else
+    cl_device_type DEVICE_TYPES = CL_DEVICE_TYPE_ALL;
+#endif
+
+    std::string deviceENV = getEnvVar("AF_OPENCL_DEVICE_TYPE");
+
+    if (deviceENV.compare("GPU") == 0) {
+        DEVICE_TYPES = CL_DEVICE_TYPE_GPU;
+    } else if (deviceENV.compare("CPU") == 0) {
+        DEVICE_TYPES = CL_DEVICE_TYPE_CPU;
+    } else if (deviceENV.compare("ACC") >= 0) {
+        DEVICE_TYPES = CL_DEVICE_TYPE_ACCELERATOR;
+    }
+
+
+
+    // Iterate through platforms, get all available devices and store them
+    for (auto &platform : platforms) {
+        std::vector<Device> current_devices;
+
+        try {
+            platform.getDevices(DEVICE_TYPES, &current_devices);
+        } catch(const cl::Error &err) {
+            if (err.err() != CL_DEVICE_NOT_FOUND) {
+                throw;
+            }
+        }
+
+        for (auto dev : current_devices) {
+            mDevices.push_back(new Device(dev));
+        }
+    }
+
+    int nDevices = mDevices.size();
+
+    if (nDevices == 0) AF_ERROR("No OpenCL devices found", AF_ERR_RUNTIME);
+
+    // Sort OpenCL devices based on default criteria
+    std::stable_sort(mDevices.begin(), mDevices.end(), compare_default);
+
+    // Create contexts and queues once the sort is done
+    for (int i = 0; i < nDevices; i++) {
+        cl_platform_id device_platform = mDevices[i]->getInfo<CL_DEVICE_PLATFORM>();
+        cl_context_properties cps[3] = {CL_CONTEXT_PLATFORM,
+                                        (cl_context_properties)(device_platform),
+                                        0};
+
+        Context *ctx = new Context(*mDevices[i], cps);
+        CommandQueue *cq = new CommandQueue(*ctx, *mDevices[i]);
+        mContexts.push_back(ctx);
+        mQueues.push_back(cq);
+        mIsGLSharingOn.push_back(false);
+        mDeviceTypes.push_back(getDeviceTypeEnum(*mDevices[i]));
+        mPlatforms.push_back(getPlatformEnum(*mDevices[i]));
+    }
+
+    bool default_device_set = false;
+    deviceENV = getEnvVar("AF_OPENCL_DEFAULT_DEVICE");
+    if(!deviceENV.empty()) {
+        std::stringstream s(deviceENV);
+        int def_device = -1;
+        s >> def_device;
+        if(def_device < 0 || def_device >= (int)nDevices) {
+            printf("WARNING: AF_OPENCL_DEFAULT_DEVICE is out of range\n");
+            printf("Setting default device as 0\n");
+        } else {
+            setContext(def_device);
+            default_device_set = true;
+        }
+    }
+
+    deviceENV = getEnvVar("AF_OPENCL_DEFAULT_DEVICE_TYPE");
+    if (!default_device_set && !deviceENV.empty())
+    {
+        cl_device_type default_device_type = CL_DEVICE_TYPE_GPU;
+        if (deviceENV.compare("CPU") == 0) {
+            default_device_type = CL_DEVICE_TYPE_CPU;
+        } else if (deviceENV.compare("ACC") >= 0) {
+            default_device_type = CL_DEVICE_TYPE_ACCELERATOR;
+        }
+
+        bool default_device_set = false;
+        for (int i = 0; i < nDevices; i++) {
+            if (mDevices[i]->getInfo<CL_DEVICE_TYPE>() == default_device_type) {
+                default_device_set = true;
+                setContext(i);
+                break;
+            }
+        }
+
+        if (!default_device_set) {
+            printf("WARNING: AF_OPENCL_DEFAULT_DEVICE_TYPE=%s is not available\n",
+                    deviceENV.c_str());
+            printf("Using default device as 0\n");
+        }
+    }
+
+#if defined(WITH_GRAPHICS)
+    // Define AF_DISABLE_GRAPHICS with any value to disable initialization
+    std::string noGraphicsENV = getEnvVar("AF_DISABLE_GRAPHICS");
+    if(noGraphicsENV.empty()) { // If AF_DISABLE_GRAPHICS is not defined
+        try {
+            /* loop over devices and replace contexts with
+             * OpenGL shared contexts whereever applicable */
+            int devCount = mDevices.size();
+            forge::Window* wHandle = graphics::ForgeManager::getInstance().getMainWindow();
+            for(int i=0; i<devCount; ++i)
+                markDeviceForInterop(i, wHandle);
+        } catch (...) {
+        }
+    }
+#endif
+    mUserDeviceOffset = mDevices.size();
+}
+
+#if defined(WITH_GRAPHICS)
+void DeviceManager::markDeviceForInterop(const int device, const forge::Window* wHandle)
+{
+    common::lock_guard_t lock(deviceMutex);
+    try {
+        if (device >= (int)mQueues.size() ||
+                device>= (int)DeviceManager::MAX_DEVICES) {
+            throw cl::Error(CL_INVALID_DEVICE, "Invalid device passed for CL-GL Interop");
+        } else {
+            mQueues[device]->finish();
+
+            // check if the device has CL_GL sharing extension enabled
+            bool temp = checkExtnAvailability(*mDevices[device], CL_GL_SHARING_EXT);
+            if (!temp) {
+                /* return silently if given device has not OpenGL sharing extension
+                 * enabled so that regular queue is used for it */
+                return;
+            }
+
+            // call forge to get OpenGL sharing context and details
+            cl::Platform plat(mDevices[device]->getInfo<CL_DEVICE_PLATFORM>());
+
+#ifdef OS_MAC
+            CGLContextObj cgl_current_ctx = CGLGetCurrentContext();
+            CGLShareGroupObj cgl_share_group = CGLGetShareGroup(cgl_current_ctx);
+
+            cl_context_properties cps[] = {
+                CL_CONTEXT_PROPERTY_USE_CGL_SHAREGROUP_APPLE, (cl_context_properties)cgl_share_group,
+                0
+            };
+#else
+            cl_context_properties cps[] = {
+                CL_GL_CONTEXT_KHR, (cl_context_properties)wHandle->context(),
+#if defined(_WIN32) || defined(_MSC_VER)
+                CL_WGL_HDC_KHR, (cl_context_properties)wHandle->display(),
+#else
+                CL_GLX_DISPLAY_KHR, (cl_context_properties)wHandle->display(),
+#endif
+                CL_CONTEXT_PLATFORM, (cl_context_properties)plat(),
+                0
+            };
+
+            // Check if current OpenCL device is belongs to the OpenGL context
+            {
+                cl_context_properties test_cps[] = {
+                    CL_GL_CONTEXT_KHR, (cl_context_properties)wHandle->context(),
+                    CL_CONTEXT_PLATFORM, (cl_context_properties)plat(),
+                    0
+                };
+
+                // Load the extension
+                // If cl_khr_gl_sharing is available, this function should be present
+                // This has been checked earlier, it comes to this point only if it is found
+                auto func = (clGetGLContextInfoKHR_fn)
+                    clGetExtensionFunctionAddressForPlatform(plat(), "clGetGLContextInfoKHR");
+
+                // If the function doesn't load, bail early
+                if (!func) return;
+
+                // Get all devices associated with opengl context
+                std::vector<cl_device_id> devices(16);
+                size_t ret = 0;
+                cl_int err = func(test_cps,
+                                  CL_DEVICES_FOR_GL_CONTEXT_KHR,
+                                  devices.size() * sizeof(cl_device_id),
+                                  &devices[0],
+                                  &ret);
+                if (err != CL_SUCCESS) return;
+                int num = ret / sizeof(cl_device_id);
+                devices.resize(num);
+
+                // Check if current device is present in the associated devices
+                cl_device_id current_device = (*mDevices[device])();
+                auto res = std::find(std::begin(devices),
+                                     std::end(devices),
+                                     current_device);
+
+                if (res == std::end(devices)) return;
+            }
+#endif
+
+            // Change current device to use GL sharing
+            Context * ctx = new Context(*mDevices[device], cps);
+            CommandQueue * cq = new CommandQueue(*ctx, *mDevices[device]);
+
+            // May be fixes the AMD GL issues we see on windows?
+#if !defined(_WIN32) && !defined(_MSC_VER)
+            delete mContexts[device];
+            delete mQueues[device];
+#endif
+
+            mContexts[device] = ctx;
+            mQueues[device] = cq;
+            mIsGLSharingOn[device] = true;
+        }
+    } catch (const cl::Error &ex) {
+        /* If replacing the original context with GL shared context
+         * failes, don't throw an error and instead fall back to
+         * original context and use copy via host to support graphics
+         * on that particular OpenCL device. So mark it as no GL sharing */
+    }
+}
+#endif
 }
 
 using namespace opencl;
diff --git a/src/backend/opencl/platform.hpp b/src/backend/opencl/platform.hpp
index 55aff48aac..846b839ee5 100644
--- a/src/backend/opencl/platform.hpp
+++ b/src/backend/opencl/platform.hpp
@@ -29,6 +29,7 @@
 #include <memory.hpp>
 #include <GraphicsResourceManager.hpp>
 #include <clfft.hpp>
+#include <common/types.hpp>
 
 namespace opencl
 {
@@ -136,6 +137,7 @@ class DeviceManager
     friend void removeDeviceContext(cl_device_id dev, cl_context ctx);
 
     friend int getActiveDeviceType();
+
     friend int getActivePlatform();
 
     public:
@@ -162,6 +164,7 @@ class DeviceManager
 
     private:
         // Attributes
+        common::mutex_t deviceMutex;
         std::vector<cl::Device*>       mDevices;
         std::vector<cl::Context*>     mContexts;
         std::vector<cl::CommandQueue*>  mQueues;
diff --git a/src/backend/opencl/types.hpp b/src/backend/opencl/types.hpp
index 5742c5d8c9..4490f2bb36 100644
--- a/src/backend/opencl/types.hpp
+++ b/src/backend/opencl/types.hpp
@@ -23,89 +23,87 @@
 
 namespace opencl
 {
+typedef cl_float2   cfloat;
+typedef cl_double2 cdouble;
+typedef cl_uchar     uchar;
+typedef cl_uint       uint;
+typedef cl_ushort   ushort;
 
-    typedef cl_float2   cfloat;
-    typedef cl_double2 cdouble;
-    typedef cl_uchar     uchar;
-    typedef cl_uint       uint;
-    typedef cl_ushort   ushort;
+template<typename T> struct is_complex          { static const bool value = false;  };
+template<> struct           is_complex<cfloat>  { static const bool value = true;   };
+template<> struct           is_complex<cdouble> { static const bool value = true;   };
 
-    template<typename T> struct is_complex          { static const bool value = false;  };
-    template<> struct           is_complex<cfloat>  { static const bool value = true;   };
-    template<> struct           is_complex<cdouble> { static const bool value = true;   };
+template<typename T > const char *shortname(bool caps=false);
 
-    template<typename T > const char *shortname(bool caps=false);
-
-    template<typename T>
-    struct ToNumStr
+template<typename T>
+struct ToNumStr
+{
+    inline std::string operator()(T val)
     {
-        inline std::string operator()(T val)
-        {
-            ToNum<T> toNum;
-            return std::to_string(toNum(val));
-        }
-    };
+        ToNum<T> toNum;
+        return std::to_string(toNum(val));
+    }
+};
 
-    template<>
-    struct ToNumStr<float>
+template<>
+struct ToNumStr<float>
+{
+    inline std::string operator()(float val)
     {
-        inline std::string operator()(float val)
-        {
-            static const std::string PINF = "+INFINITY";
-            static const std::string NINF = "-INFINITY";
-            if (std::isinf(val)) {
-                return val < 0 ? NINF : PINF;
-            }
-            return std::to_string(val);
+        static const std::string PINF = "+INFINITY";
+        static const std::string NINF = "-INFINITY";
+        if (std::isinf(val)) {
+            return val < 0 ? NINF : PINF;
         }
-    };
+        return std::to_string(val);
+    }
+};
 
-    template<>
-    struct ToNumStr<double>
+template<>
+struct ToNumStr<double>
+{
+    inline std::string operator()(double val)
     {
-        inline std::string operator()(double val)
-        {
-            static const std::string PINF = "+INFINITY";
-            static const std::string NINF = "-INFINITY";
-            if (std::isinf(val)) {
-                return val < 0 ? NINF : PINF;
-            }
-            return std::to_string(val);
+        static const std::string PINF = "+INFINITY";
+        static const std::string NINF = "-INFINITY";
+        if (std::isinf(val)) {
+            return val < 0 ? NINF : PINF;
         }
-    };
+        return std::to_string(val);
+    }
+};
 
-    template<>
-    struct ToNumStr<cfloat>
+template<>
+struct ToNumStr<cfloat>
+{
+    inline std::string operator()(cfloat val)
     {
-        inline std::string operator()(cfloat val)
-        {
-            ToNumStr<float> realStr;
-            static const std::string INF = "INFINITY";
-            std::stringstream s;
-            s << "{";
-            s << realStr(val.s[0]);
-            s << ",";
-            s << realStr(val.s[1]);
-            s << "}";
-            return s.str();
-        }
-    };
+        ToNumStr<float> realStr;
+        static const std::string INF = "INFINITY";
+        std::stringstream s;
+        s << "{";
+        s << realStr(val.s[0]);
+        s << ",";
+        s << realStr(val.s[1]);
+        s << "}";
+        return s.str();
+    }
+};
 
-    template<>
-    struct ToNumStr<cdouble>
+template<>
+struct ToNumStr<cdouble>
+{
+    inline std::string operator()(cdouble val)
     {
-        inline std::string operator()(cdouble val)
-        {
-            ToNumStr<double> realStr;
-            static const std::string INF = "INFINITY";
-            std::stringstream s;
-            s << "{";
-            s << realStr(val.s[0]);
-            s << ",";
-            s << realStr(val.s[1]);
-            s << "}";
-            return s.str();
-        }
-    };
-
+        ToNumStr<double> realStr;
+        static const std::string INF = "INFINITY";
+        std::stringstream s;
+        s << "{";
+        s << realStr(val.s[0]);
+        s << ",";
+        s << realStr(val.s[1]);
+        s << "}";
+        return s.str();
+    }
+};
 }

From 11116269d68cd00433fb326eacd9b4cd00e152e3 Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Fri, 20 Jan 2017 22:40:26 +0530
Subject: [PATCH 31/82] Make common::{InteropManager,FFTPlanCache} interfaces
 thread safe

---
 CMakeLists.txt                                |   5 +
 src/backend/common/FFTPlanCache.cpp           |  69 ++++++++
 src/backend/common/FFTPlanCache.hpp           |  72 ++------
 src/backend/common/InteropManager.cpp         | 154 ++++++++++++++++++
 src/backend/common/InteropManager.hpp         | 112 ++-----------
 src/backend/cpu/CMakeLists.txt                |   2 +
 src/backend/cuda/CMakeLists.txt               |   8 +-
 src/backend/cuda/GraphicsResourceManager.cpp  |  34 ++++
 src/backend/cuda/GraphicsResourceManager.hpp  |  25 +--
 src/backend/cuda/cufft.cpp                    |  29 ++--
 src/backend/cuda/cufft.hpp                    |  28 ++--
 src/backend/cuda/fft.cpp                      |  32 ++--
 src/backend/cuda/hist_graphics.cpp            |  12 +-
 src/backend/cuda/image.cpp                    |  13 +-
 src/backend/cuda/platform.cpp                 |   6 +-
 src/backend/cuda/plot.cpp                     |  12 +-
 src/backend/cuda/surface.cpp                  |  12 +-
 src/backend/cuda/vector_field.cpp             |  11 +-
 src/backend/opencl/CMakeLists.txt             |   7 +-
 .../opencl/GraphicsResourceManager.cpp        |  15 +-
 .../opencl/GraphicsResourceManager.hpp        |  16 +-
 src/backend/opencl/clfft.cpp                  |  73 +++------
 src/backend/opencl/clfft.hpp                  |  31 ++--
 src/backend/opencl/fft.cpp                    |  32 ++--
 src/backend/opencl/hist_graphics.cpp          |   8 +-
 src/backend/opencl/image.cpp                  |   8 +-
 src/backend/opencl/platform.cpp               |  27 ++-
 src/backend/opencl/platform.hpp               |   3 +-
 src/backend/opencl/plot.cpp                   |   8 +-
 src/backend/opencl/surface.cpp                |   8 +-
 src/backend/opencl/vector_field.cpp           |  12 +-
 31 files changed, 483 insertions(+), 401 deletions(-)
 create mode 100644 src/backend/common/FFTPlanCache.cpp
 create mode 100644 src/backend/common/InteropManager.cpp
 create mode 100644 src/backend/cuda/GraphicsResourceManager.cpp

diff --git a/CMakeLists.txt b/CMakeLists.txt
index b204310fdd..5d159c8ea5 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -53,6 +53,10 @@ ELSE(FREEIMAGE_FOUND)
     MESSAGE(WARNING, "FreeImage not found!")
 ENDIF(FREEIMAGE_FOUND)
 
+ADD_DEFINITIONS(-DBOOST_ALL_NO_LIB)
+SET(Boost_USE_STATIC_LIBS OFF)
+FIND_PACKAGE(Boost REQUIRED COMPONENTS "system" "thread")
+
 OPTION(USE_SYSTEM_CL2HPP "Use cl2.hpp installed on system" OFF)
 
 IF(BUILD_GRAPHICS)
@@ -128,6 +132,7 @@ INCLUDE_DIRECTORIES(BEFORE
     "${CMAKE_CURRENT_SOURCE_DIR}/include"
     "${CMAKE_CURRENT_SOURCE_DIR}/src/backend"
     "${CMAKE_CURRENT_SOURCE_DIR}/src/api/c"
+    "${Boost_INCLUDE_DIR}"
     )
 
 IF(${UNIX})
diff --git a/src/backend/common/FFTPlanCache.cpp b/src/backend/common/FFTPlanCache.cpp
new file mode 100644
index 0000000000..0360cb943c
--- /dev/null
+++ b/src/backend/common/FFTPlanCache.cpp
@@ -0,0 +1,69 @@
+/*******************************************************
+ * Copyright (c) 2016, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+//FIXME CPU backend doesn't required the following class implementation
+//FFTPlanCache.hpp is not used while building CPU backend.
+#ifndef AF_CPU
+#include <boost/thread.hpp>
+#include <common/FFTPlanCache.hpp>
+#include <backend.hpp>
+#include <platform.hpp>
+
+typedef boost::shared_mutex smutex_t;
+typedef boost::shared_lock<smutex_t> rlock_t;
+typedef boost::unique_lock<smutex_t> wlock_t;
+
+namespace common
+{
+static smutex_t gFFTMutexes[detail::DeviceManager::MAX_DEVICES];
+
+template<class T, typename P>
+void FFTPlanCache<T, P>::setMaxCacheSize(size_t size)
+{
+    wlock_t lock(gFFTMutexes[detail::getActiveDeviceId()]);
+    mMaxCacheSize = size;
+}
+
+template<class T, typename P>
+size_t FFTPlanCache<T, P>::getMaxCacheSize() const
+{
+    rlock_t lock(gFFTMutexes[detail::getActiveDeviceId()]);
+    return mMaxCacheSize;
+}
+
+template<class T, typename P>
+std::shared_ptr<P> FFTPlanCache<T, P>::find(const std::string& key) const
+{
+    std::shared_ptr<P> res;
+
+    rlock_t lock(gFFTMutexes[detail::getActiveDeviceId()]);
+    for(uint i=0; i<mCache.size(); ++i) {
+        if (key == mCache[i].first) {
+            res = mCache[i].second;
+            break;
+        }
+    }
+
+    return res;
+}
+
+template<class T, typename P>
+void FFTPlanCache<T, P>::push(const std::string key, std::shared_ptr<P> plan)
+{
+    wlock_t lock(gFFTMutexes[detail::getActiveDeviceId()]);
+
+    if (mCache.size()>=mMaxCacheSize)
+        mCache.pop_back();
+
+    mCache.push_front(plan_pair_t(key, plan));
+}
+
+template class FFTPlanCache<detail::PlanCache, detail::PlanType>;
+}
+#endif
diff --git a/src/backend/common/FFTPlanCache.hpp b/src/backend/common/FFTPlanCache.hpp
index 6d0699c5c1..d10baabd29 100644
--- a/src/backend/common/FFTPlanCache.hpp
+++ b/src/backend/common/FFTPlanCache.hpp
@@ -10,86 +10,46 @@
 #pragma once
 
 #include <deque>
+#include <memory>
 #include <string>
 #include <utility>
-#include <common/types.hpp>
 
 namespace common
 {
-// FFTPlanCache caches backend specific fft plans
+// FFTPlanCache caches backend specific fft plans in FIFO order
 //
-// new plan |--> IF number of plans cached is at limit, pop the least used entry and push new plan.
+// new plan |--> IF number of plans cached is at limit, pop the oldest entry and push new plan.
 //          |
 //          |--> ELSE just push the plan
 // existing plan -> reuse a plan
 template<typename T, typename P>
 class FFTPlanCache
 {
-    public:
-        FFTPlanCache() : mMaxCacheSize(5) {
-            static_cast<T*>(this)->initLibrary();
-        }
-
-        ~FFTPlanCache() {
-            static_cast<T*>(this)->deInitLibrary();
-        }
-
-        inline void maxCacheSize(size_t size) {
-            lock_guard_t lock(mutex);
-            mCache.resize(size, std::make_pair<std::string, P>(std::string(""), 0));
-        }
+    using plan_t = typename std::shared_ptr<P>;
+    using plan_pair_t = typename std::pair<std::string, plan_t>;
+    using plan_cache_t = typename std::deque<plan_pair_t>;
 
-        inline size_t maxCacheSize() const {
-            return mMaxCacheSize;
-        }
+    public:
+        FFTPlanCache() : mMaxCacheSize(5) {}
 
-        inline P get(int index) const {
-            return mCache[index].second;
-        }
+        void setMaxCacheSize(size_t size);
+        size_t getMaxCacheSize() const;
 
         // iterates through plan cache from front to back
         // of the cache(queue)
-        //
-        // A valid index of the plan in the cache is returned
-        // otherwise -1 is returned
-        int find(std::string key) const {
-            int retVal = -1;
-            for(uint i=0; i<mCache.size(); ++i) {
-                if (key == mCache[i].first) {
-                    retVal = i;
-                }
-            }
-            return retVal;
-        }
-
-        // pops plan from the back of cache(queue)
-        void pop() {
-            if (!mCache.empty()) {
-                lock_guard_t lock(mutex);
-                // destroy the cufft plan associated with the
-                // least recently used plan
-                static_cast<T*>(this)->removePlan(mCache.back().second);
-                // now pop the entry from cache
-                mCache.pop_back();
-            }
-        }
+        // A valid shared_ptr of the plan in the cache is returned
+        // if found, and empty share_ptr otherwise.
+        plan_t find(const std::string& key) const;
 
         // pushes plan to the front of cache(queue)
-        void push(std::string key, P plan) {
-            lock_guard_t lock(mutex);
-            if (mCache.size()>mMaxCacheSize) {
-                pop();
-            }
-            mCache.push_front(std::pair<std::string, P>(key, plan));
-        }
+        void push(const std::string key, plan_t plan);
 
-    private:
+    protected:
         FFTPlanCache(FFTPlanCache const&);
         void operator=(FFTPlanCache const&);
 
         size_t       mMaxCacheSize;
 
-        std::deque< std::pair<std::string, P> > mCache;
-        mutex_t mutex;
+        plan_cache_t mCache;
 };
 }
diff --git a/src/backend/common/InteropManager.cpp b/src/backend/common/InteropManager.cpp
new file mode 100644
index 0000000000..9f970e4324
--- /dev/null
+++ b/src/backend/common/InteropManager.cpp
@@ -0,0 +1,154 @@
+/*******************************************************
+ * Copyright (c) 2016, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#if defined(WITH_GRAPHICS)
+//FIXME CPU backend doesn't required the following class implementation
+//InteropManager.hpp is not used while building CPU backend.
+#ifndef AF_CPU
+#include <boost/thread.hpp>
+#include <common/InteropManager.hpp>
+#include <common/types.hpp>
+#include <cstdio>
+#include <err_common.hpp>
+#include <util.hpp>
+#include <backend.hpp>
+#include <GraphicsResourceManager.hpp>
+#include <platform.hpp>
+
+typedef boost::shared_mutex smutex_t;
+typedef boost::shared_lock<smutex_t> rlock_t;
+typedef boost::unique_lock<smutex_t> wlock_t;
+typedef boost::upgrade_lock<smutex_t> ulock_t;
+typedef boost::upgrade_to_unique_lock<smutex_t> u2ulock_t;
+
+template<typename R>
+using RVector = std::vector<std::shared_ptr<R>>;
+
+namespace common
+{
+static smutex_t gInteropMutexes[detail::DeviceManager::MAX_DEVICES];
+
+template<class T, typename R>
+InteropManager<T, R>::~InteropManager()
+{
+    try {
+        destroyResources();
+    } catch (AfError &ex) {
+
+        std::string perr = getEnvVar("AF_PRINT_ERRORS");
+        if(!perr.empty()) {
+            if(perr != "0") fprintf(stderr, "%s\n", ex.what());
+        }
+    }
+}
+
+template<class T, typename R>
+RVector<R> InteropManager<T, R>::getBufferResource(const forge::Image* image)
+{
+    ulock_t lock(gInteropMutexes[detail::getActiveDeviceId()]);
+    void * key = (void*)image;
+
+    if (mInteropMap.find(key) == mInteropMap.end()) {
+        std::vector<uint32_t> handles;
+        handles.push_back(image->pixels());
+        std::vector<resource_t> output = static_cast<T*>(this)->registerResources(handles);
+
+        u2ulock_t wlock(lock);
+        mInteropMap[key] = output;
+    }
+
+    return mInteropMap[key];
+}
+
+template<class T, typename R>
+RVector<R> InteropManager<T, R>::getBufferResource(const forge::Plot* plot)
+{
+    ulock_t lock(gInteropMutexes[detail::getActiveDeviceId()]);
+    void * key = (void*)plot;
+
+    if (mInteropMap.find(key) == mInteropMap.end()) {
+        std::vector<uint32_t> handles;
+        handles.push_back(plot->vertices());
+        std::vector<resource_t> output = static_cast<T*>(this)->registerResources(handles);
+
+        u2ulock_t wlock(lock);
+        mInteropMap[key] = output;
+    }
+
+    return mInteropMap[key];
+}
+
+template<class T, typename R>
+RVector<R> InteropManager<T, R>::getBufferResource(const forge::Histogram* histogram)
+{
+    ulock_t lock(gInteropMutexes[detail::getActiveDeviceId()]);
+    void * key = (void*)histogram;
+
+    if (mInteropMap.find(key) == mInteropMap.end()) {
+        std::vector<uint32_t> handles;
+        handles.push_back(histogram->vertices());
+        std::vector<resource_t> output = static_cast<T*>(this)->registerResources(handles);
+
+        u2ulock_t wlock(lock);
+        mInteropMap[key] = output;
+    }
+
+    return mInteropMap[key];
+}
+
+template<class T, typename R>
+RVector<R> InteropManager<T, R>::getBufferResource(const forge::Surface* surface)
+{
+    ulock_t lock(gInteropMutexes[detail::getActiveDeviceId()]);
+    void * key = (void*)surface;
+
+    if (mInteropMap.find(key) == mInteropMap.end()) {
+        std::vector<uint32_t> handles;
+        handles.push_back(surface->vertices());
+        std::vector<resource_t> output = static_cast<T*>(this)->registerResources(handles);
+
+        u2ulock_t wlock(lock);
+        mInteropMap[key] = output;
+    }
+
+    return mInteropMap[key];
+}
+
+template<class T, typename R>
+RVector<R> InteropManager<T, R>::getBufferResource(const forge::VectorField* field)
+{
+    ulock_t lock(gInteropMutexes[detail::getActiveDeviceId()]);
+    void * key = (void*)field;
+
+    if (mInteropMap.find(key) == mInteropMap.end()) {
+        std::vector<uint32_t> handles;
+        handles.push_back(field->vertices());
+        handles.push_back(field->directions());
+        std::vector<resource_t> output = static_cast<T*>(this)->registerResources(handles);
+
+        u2ulock_t wlock(lock);
+        mInteropMap[key] = output;
+    }
+
+    return mInteropMap[key];
+}
+
+template<class T, typename R>
+void InteropManager<T, R>::destroyResources()
+{
+    wlock_t lock(gInteropMutexes[detail::getActiveDeviceId()]);
+    for(auto iter : mInteropMap) {
+        iter.second.clear();
+    }
+}
+
+template class InteropManager<detail::GraphicsResourceManager, detail::CGR_t>;
+}
+#endif
+#endif
diff --git a/src/backend/common/InteropManager.hpp b/src/backend/common/InteropManager.hpp
index 6d0d6f85aa..5cfc3736e1 100644
--- a/src/backend/common/InteropManager.hpp
+++ b/src/backend/common/InteropManager.hpp
@@ -11,120 +11,34 @@
 
 #if defined(WITH_GRAPHICS)
 #include <forge.h>
-#include <err_common.hpp>
-#include <util.hpp>
-#include <cstdio>
 #include <map>
 #include <vector>
-#include <common/types.hpp>
 
 namespace common
 {
-template<typename T, typename R>
+template<class T, typename R>
 class InteropManager
 {
+    using resource_t = typename std::shared_ptr<R>;
+    using res_vec_t = typename std::vector<resource_t>;
+    using res_map_t = typename std::map<void*, res_vec_t>;
+
     public:
         InteropManager() {}
+        ~InteropManager();
 
-        ~InteropManager() {
-            try {
-                destroyResources();
-            } catch (AfError &ex) {
-
-                std::string perr = getEnvVar("AF_PRINT_ERRORS");
-                if(!perr.empty()) {
-                    if(perr != "0") fprintf(stderr, "%s\n", ex.what());
-                }
-            }
-        }
-
-        R* getBufferResource(const forge::Image* image) {
-            lock_guard_t lock(mutex);
-            void * key = (void*)image;
-
-            if (interopMap.find(key) == interopMap.end()) {
-                std::vector<uint32_t> handles;
-                handles.push_back(image->pixels());
-                std::vector<R> output = static_cast<T*>(this)->registerResources(handles);
-                interopMap[key] = output;
-            }
-
-            return &interopMap[key].front();
-        }
-
-        R* getBufferResource(const forge::Plot* plot) {
-            lock_guard_t lock(mutex);
-            void * key = (void*)plot;
-
-            if (interopMap.find(key) == interopMap.end()) {
-                std::vector<uint32_t> handles;
-                handles.push_back(plot->vertices());
-                std::vector<R> output = static_cast<T*>(this)->registerResources(handles);
-                interopMap[key] = output;
-            }
-
-            return &interopMap[key].front();
-        }
-
-        R* getBufferResource(const forge::Histogram* histogram) {
-            lock_guard_t lock(mutex);
-            void * key = (void*)histogram;
-
-            if (interopMap.find(key) == interopMap.end()) {
-                std::vector<uint32_t> handles;
-                handles.push_back(histogram->vertices());
-                std::vector<R> output = static_cast<T*>(this)->registerResources(handles);
-                interopMap[key] = output;
-            }
-
-            return &interopMap[key].front();
-        }
-
-        R* getBufferResource(const forge::Surface* surface) {
-            lock_guard_t lock(mutex);
-            void * key = (void*)surface;
-
-            if (interopMap.find(key) == interopMap.end()) {
-                std::vector<uint32_t> handles;
-                handles.push_back(surface->vertices());
-                std::vector<R> output = static_cast<T*>(this)->registerResources(handles);
-                interopMap[key] = output;
-            }
-
-            return &interopMap[key].front();
-        }
-
-        R* getBufferResource(const forge::VectorField* field) {
-            lock_guard_t lock(mutex);
-            void * key = (void*)field;
-
-            if (interopMap.find(key) == interopMap.end()) {
-                std::vector<uint32_t> handles;
-                handles.push_back(field->vertices());
-                handles.push_back(field->directions());
-                std::vector<R> output = static_cast<T*>(this)->registerResources(handles);
-                interopMap[key] = output;
-            }
-
-            return &interopMap[key].front();
-        }
+        res_vec_t getBufferResource(const forge::Image* image);
+        res_vec_t getBufferResource(const forge::Plot* plot);
+        res_vec_t getBufferResource(const forge::Histogram* histogram);
+        res_vec_t getBufferResource(const forge::Surface* surface);
+        res_vec_t getBufferResource(const forge::VectorField* field);
 
     protected:
         InteropManager(InteropManager const&);
         void operator=(InteropManager const&);
+        void destroyResources();
 
-        void destroyResources() {
-            for(auto iter : interopMap) {
-                for(auto ct : iter.second) {
-                    static_cast<T*>(this)->unregisterResource(ct);
-                }
-                iter.second.clear();
-            }
-        }
-
-        //Attributes
-        std::map<void *, std::vector<R> > interopMap;
-        mutex_t mutex;
+        res_map_t mInteropMap;
 };
 }
 #endif
diff --git a/src/backend/cpu/CMakeLists.txt b/src/backend/cpu/CMakeLists.txt
index 79c80cc26d..ab4659697f 100644
--- a/src/backend/cpu/CMakeLists.txt
+++ b/src/backend/cpu/CMakeLists.txt
@@ -126,6 +126,7 @@ FILE(GLOB backend_headers
     )
 
 FILE(GLOB backend_sources
+    "../common/*.cpp"
     "../*.cpp"
     )
 
@@ -213,6 +214,7 @@ TARGET_LINK_LIBRARIES(afcpu
                             PRIVATE ${CBLAS_LIBRARIES}
                             PRIVATE ${FFTW_LIBRARIES}
                             PRIVATE ${FreeImage_LIBS}
+                            PRIVATE ${Boost_LIBRARIES}
                      )
 
 IF(LAPACK_FOUND)
diff --git a/src/backend/cuda/CMakeLists.txt b/src/backend/cuda/CMakeLists.txt
index f5ff45092a..9b3538e576 100644
--- a/src/backend/cuda/CMakeLists.txt
+++ b/src/backend/cuda/CMakeLists.txt
@@ -188,6 +188,7 @@ FILE(GLOB backend_headers
     )
 
 FILE(GLOB backend_sources
+    "../common/*.cpp"
     "../*.cpp"
     )
 
@@ -426,14 +427,17 @@ IF (${libdevice_bc_len} GREATER 0)
   ADD_DEPENDENCIES(afcuda ${libdevice_targets})
 ENDIF()
 
-TARGET_LINK_LIBRARIES(afcuda    PRIVATE ${CUDA_CUBLAS_LIBRARIES}
+TARGET_LINK_LIBRARIES(afcuda
+                                PRIVATE ${CUDA_CUBLAS_LIBRARIES}
                                 PRIVATE ${CUDA_LIBRARIES}
                                 PRIVATE ${FreeImage_LIBS}
                                 PRIVATE ${CUDA_CUFFT_LIBRARIES}
                                 PRIVATE ${CUDA_cusparse_LIBRARY}
                                 PRIVATE ${CUDA_cusolver_LIBRARY}
                                 PRIVATE ${CUDA_nvvm_LIBRARY}
-                                PRIVATE ${CUDA_CUDA_LIBRARY})
+                                PRIVATE ${CUDA_CUDA_LIBRARY}
+                                PRIVATE ${Boost_LIBRARIES}
+                     )
 
 LIST(LENGTH GRAPHICS_DEPENDENCIES GRAPHICS_DEPENDENCIES_LEN)
 IF(${GRAPHICS_DEPENDENCIES_LEN} GREATER 0)
diff --git a/src/backend/cuda/GraphicsResourceManager.cpp b/src/backend/cuda/GraphicsResourceManager.cpp
new file mode 100644
index 0000000000..b27da053ad
--- /dev/null
+++ b/src/backend/cuda/GraphicsResourceManager.cpp
@@ -0,0 +1,34 @@
+/*******************************************************
+ * Copyright (c) 2016, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#if defined(WITH_GRAPHICS)
+#include <err_cuda.hpp>
+#include <GraphicsResourceManager.hpp>
+
+namespace cuda
+{
+ShrdResVector GraphicsResourceManager::registerResources(std::vector<uint32_t> resources)
+{
+    ShrdResVector output;
+
+    auto deleter = [](CGR_t* handle) {
+        CUDA_CHECK(cudaGraphicsUnregisterResource(*handle));
+        delete handle;
+    };
+
+    for (auto id: resources) {
+        CGR_t r;
+        CUDA_CHECK(cudaGraphicsGLRegisterBuffer(&r, id, cudaGraphicsMapFlagsWriteDiscard));
+        output.emplace_back(new CGR_t(r), deleter);
+    }
+
+    return output;
+}
+}
+#endif
diff --git a/src/backend/cuda/GraphicsResourceManager.hpp b/src/backend/cuda/GraphicsResourceManager.hpp
index 6f8b39fd8e..fb0700d441 100644
--- a/src/backend/cuda/GraphicsResourceManager.hpp
+++ b/src/backend/cuda/GraphicsResourceManager.hpp
@@ -14,42 +14,29 @@
 #include <windows.h>
 #endif
 
-#include <platform.hpp>
 // cuda_gl_interop.h does not include OpenGL headers for ARM
 #include <graphics_common.hpp>
 using namespace gl;
 #define GL_VERSION gl::GL_VERSION
-#define __gl_h_ //Hack to avoid gl.h inclusion by cuda_gl_interop.h
+#define __gl_h_ //FIXME Hack to avoid gl.h inclusion by cuda_gl_interop.h
+#include <cuda.h>
+#include <cuda_runtime.h>
 #include <cuda_gl_interop.h>
 #include <common/InteropManager.hpp>
-#include <err_cuda.hpp>
 #include <map>
 #include <vector>
 
 namespace cuda
 {
 typedef cudaGraphicsResource_t CGR_t;
+typedef std::shared_ptr<CGR_t> SharedResource;
+typedef std::vector<SharedResource> ShrdResVector;
 
 class GraphicsResourceManager : public common::InteropManager<GraphicsResourceManager, CGR_t>
 {
     public:
         GraphicsResourceManager() {}
-
-        std::vector<CGR_t> registerResources(std::vector<uint32_t> resources) {
-            std::vector<CGR_t> output;
-
-            for (auto id: resources) {
-                CGR_t r;
-                CUDA_CHECK(cudaGraphicsGLRegisterBuffer(&r, id, cudaGraphicsMapFlagsWriteDiscard));
-                output.push_back(r);
-            }
-
-            return output;
-        }
-
-        void unregisterResource(CGR_t handle) {
-            CUDA_CHECK(cudaGraphicsUnregisterResource(handle));
-        }
+        ShrdResVector registerResources(std::vector<uint32_t> resources);
 
     protected:
         GraphicsResourceManager(GraphicsResourceManager const&);
diff --git a/src/backend/cuda/cufft.cpp b/src/backend/cuda/cufft.cpp
index b185486158..727c693f2a 100644
--- a/src/backend/cuda/cufft.cpp
+++ b/src/backend/cuda/cufft.cpp
@@ -74,10 +74,10 @@ const char * _cufftGetResultString(cufftResult res)
     return "cuFFT: unknown error";
 }
 
-PlanType findPlan(int rank, int *n,
-                  int *inembed, int istride, int idist,
-                  int *onembed, int ostride, int odist,
-                  cufftType type, int batch)
+SharedPlan findPlan(int rank, int *n,
+                    int *inembed, int istride, int idist,
+                    int *onembed, int ostride, int odist,
+                    cufftType type, int batch)
 {
     // create the key string
     char key_str_temp[64];
@@ -112,29 +112,30 @@ PlanType findPlan(int rank, int *n,
     key_string.append(std::string(key_str_temp));
 
     PlanCache &planner = cuda::fftManager();
+    SharedPlan retVal  = planner.find(key_string);
 
-    int planIndex = planner.find(key_string);
+    if (retVal)
+        return retVal;
 
-    // if found a valid plan, return it
-    if (planIndex!=-1) {
-        return planner.get(planIndex);
-    }
-
-    PlanType retVal;
-    cufftResult res = cufftPlanMany(&retVal, rank, n,
+    PlanType* temp = (PlanType*)malloc(sizeof(PlanType));
+    cufftResult res = cufftPlanMany(temp, rank, n,
                                     inembed, istride, idist, onembed, ostride, odist,
                                     type, batch);
 
     // If plan creation fails, clean up the memory we hold on to and try again
     if (res != CUFFT_SUCCESS) {
         cuda::garbageCollect();
-        CUFFT_CHECK(cufftPlanMany(&retVal, rank, n,
+        CUFFT_CHECK(cufftPlanMany(temp, rank, n,
                                   inembed, istride, idist, onembed, ostride, odist,
                                   type, batch));
     }
 
-    cufftSetStream(retVal, cuda::getActiveStream());
+    cufftSetStream(*temp, cuda::getActiveStream());
 
+    retVal.reset(temp, [](PlanType* p) {
+        cufftDestroy(*p);
+        delete p;
+    });
     // push the plan into plan cache
     planner.push(key_string, retVal);
 
diff --git a/src/backend/cuda/cufft.hpp b/src/backend/cuda/cufft.hpp
index 6142004151..e4f4326000 100644
--- a/src/backend/cuda/cufft.hpp
+++ b/src/backend/cuda/cufft.hpp
@@ -15,30 +15,22 @@
 
 namespace cuda
 {
-typedef cufftHandle PlanType; //used in platform.hpp
+typedef cufftHandle PlanType;
+typedef std::shared_ptr<PlanType> SharedPlan;
 
 const char * _cufftGetResultString(cufftResult res);
 
-PlanType findPlan(int rank, int *n,
-                  int *inembed, int istride, int idist,
-                  int *onembed, int ostride, int odist,
-                  cufftType type, int batch);
+SharedPlan findPlan(int rank, int *n,
+                    int *inembed, int istride, int idist,
+                    int *onembed, int ostride, int odist,
+                    cufftType type, int batch);
 
 class PlanCache : public common::FFTPlanCache<PlanCache, PlanType>
 {
-    friend PlanType findPlan(int rank, int *n,
-                             int *inembed, int istride, int idist,
-                             int *onembed, int ostride, int odist,
-                             cufftType type, int batch);
-
-    public:
-        PlanCache() {}
-        void initLibrary() {}
-        void deInitLibrary() {}
-
-        void removePlan(PlanType plan) {
-            cufftDestroy(plan);
-        }
+    friend SharedPlan findPlan(int rank, int *n,
+                               int *inembed, int istride, int idist,
+                               int *onembed, int ostride, int odist,
+                               cufftType type, int batch);
 };
 }
 
diff --git a/src/backend/cuda/fft.cpp b/src/backend/cuda/fft.cpp
index ca41896e29..280704012a 100644
--- a/src/backend/cuda/fft.cpp
+++ b/src/backend/cuda/fft.cpp
@@ -23,7 +23,7 @@ namespace cuda
 {
 void setFFTPlanCacheSize(size_t numPlans)
 {
-    fftManager().maxCacheSize(numPlans);
+    fftManager().setMaxCacheSize(numPlans);
 }
 
 template<typename T>
@@ -88,13 +88,13 @@ void fft_inplace(Array<T> &in)
         batch *= idims[i];
     }
 
-    cufftHandle plan = findPlan(rank, t_dims,
-                                in_embed , istrides[0], istrides[rank],
-                                in_embed , istrides[0], istrides[rank],
-                                (cufftType)cufft_transform<T>::type, batch);
+    SharedPlan plan = findPlan(rank, t_dims,
+                               in_embed , istrides[0], istrides[rank],
+                               in_embed , istrides[0], istrides[rank],
+                               (cufftType)cufft_transform<T>::type, batch);
 
     cufft_transform<T> transform;
-    CUFFT_CHECK(transform(plan, (T *)in.get(), in.get(), direction ? CUFFT_FORWARD : CUFFT_INVERSE));
+    CUFFT_CHECK(transform(*plan.get(), (T *)in.get(), in.get(), direction ? CUFFT_FORWARD : CUFFT_INVERSE));
 }
 
 template<typename Tc, typename Tr, int rank>
@@ -122,13 +122,13 @@ Array<Tc> fft_r2c(const Array<Tr> &in)
     dim4 istrides = in.strides();
     dim4 ostrides = out.strides();
 
-    cufftHandle plan = findPlan(rank, t_dims,
-                                in_embed  , istrides[0], istrides[rank],
-                                out_embed , ostrides[0], ostrides[rank],
-                                (cufftType)cufft_real_transform<Tc, Tr>::type, batch);
+    SharedPlan plan = findPlan(rank, t_dims,
+                               in_embed  , istrides[0], istrides[rank],
+                               out_embed , ostrides[0], ostrides[rank],
+                               (cufftType)cufft_real_transform<Tc, Tr>::type, batch);
 
     cufft_real_transform<Tc, Tr> transform;
-    CUFFT_CHECK(transform(plan, (Tr *)in.get(), out.get()));
+    CUFFT_CHECK(transform(*plan.get(), (Tr *)in.get(), out.get()));
     return out;
 }
 
@@ -154,12 +154,12 @@ Array<Tr> fft_c2r(const Array<Tc> &in, const dim4 &odims)
 
     cufft_real_transform<Tr, Tc> transform;
 
-    cufftHandle plan = findPlan(rank, t_dims,
-                                in_embed  , istrides[0], istrides[rank],
-                                out_embed , ostrides[0], ostrides[rank],
-                                (cufftType)cufft_real_transform<Tr, Tc>::type, batch);
+    SharedPlan plan = findPlan(rank, t_dims,
+                               in_embed  , istrides[0], istrides[rank],
+                               out_embed , ostrides[0], ostrides[rank],
+                               (cufftType)cufft_real_transform<Tr, Tc>::type, batch);
 
-    CUFFT_CHECK(transform(plan, (Tc *)in.get(), out.get()));
+    CUFFT_CHECK(transform(*plan.get(), (Tc *)in.get(), out.get()));
     return out;
 }
 
diff --git a/src/backend/cuda/hist_graphics.cpp b/src/backend/cuda/hist_graphics.cpp
index 6367208c5c..d43355a123 100644
--- a/src/backend/cuda/hist_graphics.cpp
+++ b/src/backend/cuda/hist_graphics.cpp
@@ -25,17 +25,15 @@ void copy_histogram(const Array<T> &data, const forge::Histogram* hist)
     if(DeviceManager::checkGraphicsInteropCapability()) {
         const T *d_P = data.get();
 
-        GraphicsResourceManager& intrpMngr = interopManager();
+        ShrdResVector res = interopManager().getBufferResource(hist);
 
-        cudaGraphicsResource_t *resources = intrpMngr.getBufferResource(hist);
         // Map resource. Copy data to VBO. Unmap resource.
         size_t num_bytes = hist->verticesSize();
         T* d_vbo = NULL;
-        cudaGraphicsMapResources(1, resources, cuda::getActiveStream());
-        cudaGraphicsResourceGetMappedPointer((void **)&d_vbo, &num_bytes, resources[0]);
-        cudaMemcpyAsync(d_vbo, d_P, num_bytes, cudaMemcpyDeviceToDevice,
-                        cuda::getActiveStream());
-        cudaGraphicsUnmapResources(1, resources, cuda::getActiveStream());
+        cudaGraphicsMapResources(1, res[0].get(), cuda::getActiveStream());
+        cudaGraphicsResourceGetMappedPointer((void **)&d_vbo, &num_bytes, *(res[0].get()));
+        cudaMemcpyAsync(d_vbo, d_P, num_bytes, cudaMemcpyDeviceToDevice, cuda::getActiveStream());
+        cudaGraphicsUnmapResources(1, res[0].get(), cuda::getActiveStream());
 
         CheckGL("After cuda resource copy");
 
diff --git a/src/backend/cuda/image.cpp b/src/backend/cuda/image.cpp
index 17d0f12b0f..ef177dc28f 100644
--- a/src/backend/cuda/image.cpp
+++ b/src/backend/cuda/image.cpp
@@ -28,19 +28,16 @@ template<typename T>
 void copy_image(const Array<T> &in, const forge::Image* image)
 {
     if(DeviceManager::checkGraphicsInteropCapability()) {
-        GraphicsResourceManager& intrpMngr = interopManager();
-
-        cudaGraphicsResource_t *resources = intrpMngr.getBufferResource(image);
+        ShrdResVector res = interopManager().getBufferResource(image);
 
         const T *d_X = in.get();
         // Map resource. Copy data to pixels. Unmap resource.
         size_t num_bytes;
         T* d_pixels = NULL;
-        cudaGraphicsMapResources(1, resources, cuda::getActiveStream());
-        cudaGraphicsResourceGetMappedPointer((void **)&d_pixels, &num_bytes, resources[0]);
-        cudaMemcpyAsync(d_pixels, d_X, num_bytes, cudaMemcpyDeviceToDevice,
-                        cuda::getActiveStream());
-        cudaGraphicsUnmapResources(1, resources, cuda::getActiveStream());
+        cudaGraphicsMapResources(1, res[0].get(), cuda::getActiveStream());
+        cudaGraphicsResourceGetMappedPointer((void **)&d_pixels, &num_bytes, *(res[0].get()));
+        cudaMemcpyAsync(d_pixels, d_X, num_bytes, cudaMemcpyDeviceToDevice, cuda::getActiveStream());
+        cudaGraphicsUnmapResources(1, res[0].get(), cuda::getActiveStream());
 
         POST_LAUNCH_CHECK();
         CheckGL("After cuda resource copy");
diff --git a/src/backend/cuda/platform.cpp b/src/backend/cuda/platform.cpp
index 0c19208311..ce55bca8eb 100644
--- a/src/backend/cuda/platform.cpp
+++ b/src/backend/cuda/platform.cpp
@@ -475,7 +475,7 @@ SparseHandle sparseHandle()
 
     std::call_once(initFlags[id], [&]{ inst.cusparseHandles[id].reset(new cusparseHandle()); });
 
-    return instance.cusparseHandles[id].get()->get();
+    return inst.cusparseHandles[id].get()->get();
 }
 
 DeviceManager::DeviceManager()
@@ -521,7 +521,7 @@ DeviceManager::DeviceManager()
 
 void DeviceManager::sortDevices(sort_mode mode)
 {
-    lock_guard_t lock(deviceMutex);
+    common::lock_guard_t lock(deviceMutex);
     switch(mode) {
         case memory :
             std::stable_sort(cuDevices.begin(), cuDevices.end(), card_compare_mem);
@@ -542,7 +542,7 @@ int DeviceManager::setActiveDevice(int device, int nId)
 {
     static bool first = true;
 
-    lock_guard_t lock(deviceMutex);
+    common::lock_guard_t lock(deviceMutex);
 
     int numDevices = cuDevices.size();
 
diff --git a/src/backend/cuda/plot.cpp b/src/backend/cuda/plot.cpp
index e7a31537ff..e9d7bea738 100644
--- a/src/backend/cuda/plot.cpp
+++ b/src/backend/cuda/plot.cpp
@@ -30,17 +30,15 @@ void copy_plot(const Array<T> &P, forge::Plot* plot)
     if(DeviceManager::checkGraphicsInteropCapability()) {
         const T *d_P = P.get();
 
-        GraphicsResourceManager& intrpMngr = interopManager();
+        ShrdResVector res = interopManager().getBufferResource(plot);
 
-        cudaGraphicsResource_t *resources = intrpMngr.getBufferResource(plot);
         // Map resource. Copy data to VBO. Unmap resource.
         size_t num_bytes = plot->verticesSize();
         T* d_vbo = NULL;
-        cudaGraphicsMapResources(1, resources, cuda::getActiveStream());
-        cudaGraphicsResourceGetMappedPointer((void **)&d_vbo, &num_bytes, resources[0]);
-        cudaMemcpyAsync(d_vbo, d_P, num_bytes, cudaMemcpyDeviceToDevice,
-                cuda::getActiveStream());
-        cudaGraphicsUnmapResources(1, resources, cuda::getActiveStream());
+        cudaGraphicsMapResources(1, res[0].get(), cuda::getActiveStream());
+        cudaGraphicsResourceGetMappedPointer((void **)&d_vbo, &num_bytes, *(res[0].get()));
+        cudaMemcpyAsync(d_vbo, d_P, num_bytes, cudaMemcpyDeviceToDevice, cuda::getActiveStream());
+        cudaGraphicsUnmapResources(1, res[0].get(), cuda::getActiveStream());
 
         CheckGL("After cuda resource copy");
 
diff --git a/src/backend/cuda/surface.cpp b/src/backend/cuda/surface.cpp
index 650968a61f..cc7ac29e73 100644
--- a/src/backend/cuda/surface.cpp
+++ b/src/backend/cuda/surface.cpp
@@ -30,17 +30,15 @@ void copy_surface(const Array<T> &P, forge::Surface* surface)
     if(DeviceManager::checkGraphicsInteropCapability()) {
         const T *d_P = P.get();
 
-        GraphicsResourceManager& intrpMngr = interopManager();
+        ShrdResVector res = interopManager().getBufferResource(surface);
 
-        cudaGraphicsResource_t *resources = intrpMngr.getBufferResource(surface);
         // Map resource. Copy data to VBO. Unmap resource.
         size_t num_bytes = surface->verticesSize();
         T* d_vbo = NULL;
-        cudaGraphicsMapResources(1, resources, cuda::getActiveStream());
-        cudaGraphicsResourceGetMappedPointer((void **)&d_vbo, &num_bytes, resources[0]);
-        cudaMemcpyAsync(d_vbo, d_P, num_bytes, cudaMemcpyDeviceToDevice,
-                cuda::getActiveStream());
-        cudaGraphicsUnmapResources(1, resources, cuda::getActiveStream());
+        cudaGraphicsMapResources(1, res[0].get(), cuda::getActiveStream());
+        cudaGraphicsResourceGetMappedPointer((void **)&d_vbo, &num_bytes, *(res[0].get()));
+        cudaMemcpyAsync(d_vbo, d_P, num_bytes, cudaMemcpyDeviceToDevice, cuda::getActiveStream());
+        cudaGraphicsUnmapResources(1, res[0].get(), cuda::getActiveStream());
 
         CheckGL("After cuda resource copy");
 
diff --git a/src/backend/cuda/vector_field.cpp b/src/backend/cuda/vector_field.cpp
index c875130a8b..043d0ba591 100644
--- a/src/backend/cuda/vector_field.cpp
+++ b/src/backend/cuda/vector_field.cpp
@@ -26,9 +26,8 @@ void copy_vector_field(const Array<T> &points, const Array<T> &directions,
                        forge::VectorField* vector_field)
 {
     if(DeviceManager::checkGraphicsInteropCapability()) {
-        GraphicsResourceManager& intrpMngr = interopManager();
-
-        cudaGraphicsResource_t *resources = intrpMngr.getBufferResource(vector_field);
+        ShrdResVector res = interopManager().getBufferResource(vector_field);
+        CGR_t resources[2] = {*res[0].get(), *res[1].get()};
 
         // Map resource. Copy data to VBO. Unmap resource.
         // Map all resources at once.
@@ -40,8 +39,7 @@ void copy_vector_field(const Array<T> &points, const Array<T> &directions,
             size_t num_bytes = vector_field->verticesSize();
             T* d_vbo = NULL;
             cudaGraphicsResourceGetMappedPointer((void **)&d_vbo, &num_bytes, resources[0]);
-            cudaMemcpyAsync(d_vbo, ptr, num_bytes, cudaMemcpyDeviceToDevice,
-                            cuda::getActiveStream());
+            cudaMemcpyAsync(d_vbo, ptr, num_bytes, cudaMemcpyDeviceToDevice, cuda::getActiveStream());
         }
         // Directions
         {
@@ -49,8 +47,7 @@ void copy_vector_field(const Array<T> &points, const Array<T> &directions,
             size_t num_bytes = vector_field->directionsSize();
             T* d_vbo = NULL;
             cudaGraphicsResourceGetMappedPointer((void **)&d_vbo, &num_bytes, resources[1]);
-            cudaMemcpyAsync(d_vbo, ptr, num_bytes, cudaMemcpyDeviceToDevice,
-                            cuda::getActiveStream());
+            cudaMemcpyAsync(d_vbo, ptr, num_bytes, cudaMemcpyDeviceToDevice, cuda::getActiveStream());
         }
         cudaGraphicsUnmapResources(2, resources, cuda::getActiveStream());
 
diff --git a/src/backend/opencl/CMakeLists.txt b/src/backend/opencl/CMakeLists.txt
index 37655cfd36..de2b9ec588 100644
--- a/src/backend/opencl/CMakeLists.txt
+++ b/src/backend/opencl/CMakeLists.txt
@@ -78,10 +78,6 @@ ENDIF()
 INCLUDE_DIRECTORIES(${CLFFT_INCLUDE_DIRS})
 LINK_DIRECTORIES(${CLFFT_LIBRARY_DIR})
 
-ADD_DEFINITIONS( -DBOOST_ALL_NO_LIB )
-SET(Boost_USE_STATIC_LIBS OFF)
-FIND_PACKAGE(Boost 1.48 REQUIRED)
-
 OPTION(USE_SYSTEM_BOOST_COMPUTE "Use system BoostCompute" OFF)
 IF(USE_SYSTEM_BOOST_COMPUTE)
     IF(Boost_VERSION VERSION_LESS "1.61")
@@ -102,7 +98,6 @@ INCLUDE_DIRECTORIES(
     "${CMAKE_CURRENT_BINARY_DIR}"
     ${CLBLAS_INCLUDE_DIRS}
     ${CLFFT_INCLUDE_DIRS}
-    ${Boost_INCLUDE_DIR}
     ${BoostCompute_INCLUDE_DIRS}
     ${CBLAS_INCLUDE_DIR}
     )
@@ -186,6 +181,7 @@ FILE(GLOB backend_headers
     )
 
 FILE(GLOB backend_sources
+    "../common/*.cpp"
     "../*.cpp"
     )
 
@@ -323,6 +319,7 @@ TARGET_LINK_LIBRARIES(afopencl
                                 PRIVATE ${CLFFT_LIBRARIES}
                                 PRIVATE ${CMAKE_DL_LIBS}
                                 PRIVATE ${FreeImage_LIBS}
+                                PRIVATE ${Boost_LIBRARIES}
                      )
 
 IF(LAPACK_FOUND)
diff --git a/src/backend/opencl/GraphicsResourceManager.cpp b/src/backend/opencl/GraphicsResourceManager.cpp
index 8e043313ed..77d20ee9b9 100644
--- a/src/backend/opencl/GraphicsResourceManager.cpp
+++ b/src/backend/opencl/GraphicsResourceManager.cpp
@@ -13,21 +13,14 @@
 
 namespace opencl
 {
-std::vector<CGR_t> GraphicsResourceManager::registerResources(std::vector<uint32_t> resources)
+ShrdResVector GraphicsResourceManager::registerResources(std::vector<uint32_t> resources)
 {
-    std::vector<CGR_t> output;
+    ShrdResVector output;
 
-    for (auto id: resources) {
-        CGR_t r = new cl::BufferGL(opencl::getContext(), CL_MEM_WRITE_ONLY, id, NULL);
-        output.push_back(r);
-    }
+    for (auto id: resources)
+        output.emplace_back(new cl::BufferGL(getContext(), CL_MEM_WRITE_ONLY, id, NULL));
 
     return output;
 }
-
-void GraphicsResourceManager::unregisterResource(CGR_t handle)
-{
-    delete handle;
-}
 }
 #endif
diff --git a/src/backend/opencl/GraphicsResourceManager.hpp b/src/backend/opencl/GraphicsResourceManager.hpp
index 774cb0f124..d498dab894 100644
--- a/src/backend/opencl/GraphicsResourceManager.hpp
+++ b/src/backend/opencl/GraphicsResourceManager.hpp
@@ -10,22 +10,26 @@
 #pragma once
 
 #if defined(WITH_GRAPHICS)
-#include <platform.hpp>
 #include <common/InteropManager.hpp>
 #include <map>
 #include <vector>
 
+namespace cl
+{
+class Buffer;
+}
+
 namespace opencl
 {
-typedef cl::Buffer* CGR_t;
+typedef cl::Buffer CGR_t;
+typedef std::shared_ptr<CGR_t> SharedResource;
+typedef std::vector<SharedResource> ShrdResVector;
 
-class GraphicsResourceManager : public common::InteropManager<GraphicsResourceManager, CGR_t>
+class GraphicsResourceManager : public common::InteropManager<GraphicsResourceManager, cl::Buffer>
 {
     public:
         GraphicsResourceManager() {}
-
-        std::vector<CGR_t> registerResources(std::vector<uint32_t> resources);
-        void unregisterResource(CGR_t handle);
+        ShrdResVector registerResources(std::vector<uint32_t> resources);
 
     protected:
         GraphicsResourceManager(GraphicsResourceManager const&);
diff --git a/src/backend/opencl/clfft.cpp b/src/backend/opencl/clfft.cpp
index 483d88cfd6..aa7d26e63d 100644
--- a/src/backend/opencl/clfft.cpp
+++ b/src/backend/opencl/clfft.cpp
@@ -83,11 +83,11 @@ const char * _clfftGetResultString(clfftStatus st)
     return "Unknown error";
 }
 
-PlanType findPlan(clfftLayout iLayout, clfftLayout oLayout,
-                  clfftDim rank, size_t *clLengths,
-                  size_t *istrides, size_t idist,
-                  size_t *ostrides, size_t odist,
-                  clfftPrecision precision, size_t batch)
+SharedPlan findPlan(clfftLayout iLayout, clfftLayout oLayout,
+                    clfftDim rank, size_t *clLengths,
+                    size_t *istrides, size_t idist,
+                    size_t *ostrides, size_t odist,
+                    clfftPrecision precision, size_t batch)
 {
     // create the key string
     char key_str_temp[64];
@@ -123,68 +123,43 @@ PlanType findPlan(clfftLayout iLayout, clfftLayout oLayout,
     key_string.append(std::string(key_str_temp));
 
     PlanCache &planner = opencl::fftManager();
+    SharedPlan retVal = planner.find(key_string);
 
-    int planIndex = planner.find(key_string);
+    if (retVal)
+        return retVal;
 
-    // if found a valid plan, return it
-    if (planIndex!=-1) {
-        return planner.get(planIndex);
-    }
-
-    PlanType retVal;
+    PlanType* temp = (PlanType*)malloc(sizeof(PlanType));
 
     // getContext() returns object of type Context
     // Context() returns the actual cl_context handle
-    CLFFT_CHECK(clfftCreateDefaultPlan(&retVal, opencl::getContext()(), rank, clLengths));
+    CLFFT_CHECK(clfftCreateDefaultPlan(temp, opencl::getContext()(), rank, clLengths));
 
     // complex to complex
     if (iLayout == oLayout) {
-        CLFFT_CHECK(clfftSetResultLocation(retVal, CLFFT_INPLACE));
+        CLFFT_CHECK(clfftSetResultLocation(*temp, CLFFT_INPLACE));
     } else {
-        CLFFT_CHECK(clfftSetResultLocation(retVal, CLFFT_OUTOFPLACE));
+        CLFFT_CHECK(clfftSetResultLocation(*temp, CLFFT_OUTOFPLACE));
     }
 
-    CLFFT_CHECK(clfftSetLayout(retVal, iLayout, oLayout));
-    CLFFT_CHECK(clfftSetPlanBatchSize(retVal, batch));
-    CLFFT_CHECK(clfftSetPlanDistance(retVal, idist, odist));
-    CLFFT_CHECK(clfftSetPlanInStride(retVal, rank, istrides));
-    CLFFT_CHECK(clfftSetPlanOutStride(retVal, rank, ostrides));
-    CLFFT_CHECK(clfftSetPlanPrecision(retVal, precision));
-    CLFFT_CHECK(clfftSetPlanScale(retVal, CLFFT_BACKWARD, 1.0));
+    CLFFT_CHECK(clfftSetLayout(*temp, iLayout, oLayout));
+    CLFFT_CHECK(clfftSetPlanBatchSize(*temp, batch));
+    CLFFT_CHECK(clfftSetPlanDistance(*temp, idist, odist));
+    CLFFT_CHECK(clfftSetPlanInStride(*temp, rank, istrides));
+    CLFFT_CHECK(clfftSetPlanOutStride(*temp, rank, ostrides));
+    CLFFT_CHECK(clfftSetPlanPrecision(*temp, precision));
+    CLFFT_CHECK(clfftSetPlanScale(*temp, CLFFT_BACKWARD, 1.0));
 
     // getQueue() returns object of type CommandQueue
     // CommandQueue() returns the actual cl_command_queue handle
-    CLFFT_CHECK(clfftBakePlan(retVal, 1, &(opencl::getQueue()()), NULL, NULL));
+    CLFFT_CHECK(clfftBakePlan(*temp, 1, &(opencl::getQueue()()), NULL, NULL));
 
+    retVal.reset(temp, [](PlanType* p) {
+        CLFFT_CHECK(clfftDestroyPlan(p));
+        free(p);
+    });
     // push the plan into plan cache
     planner.push(key_string, retVal);
 
     return retVal;
 }
-
-void PlanCache::initLibrary()
-{
-    CLFFT_CHECK(clfftInitSetupData(&mFFTSetup));
-    CLFFT_CHECK(clfftSetup(&mFFTSetup));
-}
-
-void PlanCache::deInitLibrary()
-{
-    //TODO: FIXME:
-    // clfftTeardown() causes a "Pure Virtual Function Called" crash on
-    // Windows for Intel devices. This causes tests to fail.
-    #ifndef OS_WIN
-    static bool flag = true;
-    if(flag) {
-        // THOU SHALL NOT THROW IN DESTRUCTORS
-        clfftTeardown();
-        flag = false;
-    }
-    #endif
-}
-
-void PlanCache::removePlan(PlanType plan)
-{
-     CLFFT_CHECK(clfftDestroyPlan(&plan));
-}
 }
diff --git a/src/backend/opencl/clfft.hpp b/src/backend/opencl/clfft.hpp
index 92267e27c7..f249735712 100644
--- a/src/backend/opencl/clfft.hpp
+++ b/src/backend/opencl/clfft.hpp
@@ -17,32 +17,23 @@
 namespace opencl
 {
 typedef clfftPlanHandle PlanType;
+typedef std::shared_ptr<PlanType> SharedPlan;
 
 const char * _clfftGetResultString(clfftStatus st);
 
-PlanType findPlan(clfftLayout iLayout, clfftLayout oLayout,
-                  clfftDim rank, size_t *clLengths,
-                  size_t *istrides, size_t idist,
-                  size_t *ostrides, size_t odist,
-                  clfftPrecision precision, size_t batch);
+SharedPlan findPlan(clfftLayout iLayout, clfftLayout oLayout,
+                    clfftDim rank, size_t *clLengths,
+                    size_t *istrides, size_t idist,
+                    size_t *ostrides, size_t odist,
+                    clfftPrecision precision, size_t batch);
 
 class PlanCache : public common::FFTPlanCache<PlanCache, PlanType>
 {
-    friend PlanType findPlan(clfftLayout iLayout, clfftLayout oLayout,
-                             clfftDim rank, size_t *clLengths,
-                             size_t *istrides, size_t idist,
-                             size_t *ostrides, size_t odist,
-                             clfftPrecision precision, size_t batch);
-
-    public:
-        PlanCache() {}
-        void initLibrary();
-        void deInitLibrary();
-
-        void removePlan(PlanType plan);
-
-    private:
-        clfftSetupData  mFFTSetup;
+    friend SharedPlan findPlan(clfftLayout iLayout, clfftLayout oLayout,
+                               clfftDim rank, size_t *clLengths,
+                               size_t *istrides, size_t idist,
+                               size_t *ostrides, size_t odist,
+                               clfftPrecision precision, size_t batch);
 };
 }
 
diff --git a/src/backend/opencl/fft.cpp b/src/backend/opencl/fft.cpp
index 56a1927d84..b882b21cdf 100644
--- a/src/backend/opencl/fft.cpp
+++ b/src/backend/opencl/fft.cpp
@@ -24,7 +24,7 @@ namespace opencl
 
 void setFFTPlanCacheSize(size_t numPlans)
 {
-    fftManager().maxCacheSize(numPlans);
+    fftManager().setMaxCacheSize(numPlans);
 }
 
 template<typename T> struct Precision;
@@ -83,15 +83,15 @@ void fft_inplace(Array<T> &in)
         batch *= tdims[i];
     }
 
-    PlanType plan = findPlan(CLFFT_COMPLEX_INTERLEAVED, CLFFT_COMPLEX_INTERLEAVED,
-                             (clfftDim)rank, tdims,
-                             istrides, istrides[rank], istrides, istrides[rank],
-                             (clfftPrecision)Precision<T>::type, batch);
+    SharedPlan plan = findPlan(CLFFT_COMPLEX_INTERLEAVED, CLFFT_COMPLEX_INTERLEAVED,
+                               (clfftDim)rank, tdims,
+                               istrides, istrides[rank], istrides, istrides[rank],
+                               (clfftPrecision)Precision<T>::type, batch);
 
     cl_mem imem = (*in.get())();
     cl_command_queue queue = getQueue()();
 
-    CLFFT_CHECK(clfftEnqueueTransform(plan,
+    CLFFT_CHECK(clfftEnqueueTransform(*plan.get(),
                                       direction ? CLFFT_FORWARD : CLFFT_BACKWARD,
                                       1, &queue, 0, NULL, NULL,
                                       &imem, &imem, NULL));
@@ -118,16 +118,16 @@ Array<Tc> fft_r2c(const Array<Tr> &in)
         batch *= tdims[i];
     }
 
-    PlanType plan = findPlan(CLFFT_REAL, CLFFT_HERMITIAN_INTERLEAVED,
-                             (clfftDim)rank, tdims,
-                             istrides, istrides[rank], ostrides, ostrides[rank],
-                             (clfftPrecision)Precision<Tc>::type, batch);
+    SharedPlan plan = findPlan(CLFFT_REAL, CLFFT_HERMITIAN_INTERLEAVED,
+                               (clfftDim)rank, tdims,
+                               istrides, istrides[rank], ostrides, ostrides[rank],
+                               (clfftPrecision)Precision<Tc>::type, batch);
 
     cl_mem imem = (*in.get())();
     cl_mem omem = (*out.get())();
     cl_command_queue queue = getQueue()();
 
-    CLFFT_CHECK(clfftEnqueueTransform(plan,
+    CLFFT_CHECK(clfftEnqueueTransform(*plan.get(),
                                       CLFFT_FORWARD,
                                       1, &queue, 0, NULL, NULL,
                                       &imem, &omem, NULL));
@@ -152,16 +152,16 @@ Array<Tr> fft_c2r(const Array<Tc> &in, const dim4 &odims)
         batch *= tdims[i];
     }
 
-    PlanType plan = findPlan(CLFFT_HERMITIAN_INTERLEAVED, CLFFT_REAL,
-                             (clfftDim)rank, tdims,
-                             istrides, istrides[rank], ostrides, ostrides[rank],
-                             (clfftPrecision)Precision<Tc>::type, batch);
+    SharedPlan plan = findPlan(CLFFT_HERMITIAN_INTERLEAVED, CLFFT_REAL,
+                               (clfftDim)rank, tdims,
+                               istrides, istrides[rank], ostrides, ostrides[rank],
+                               (clfftPrecision)Precision<Tc>::type, batch);
 
     cl_mem imem = (*in.get())();
     cl_mem omem = (*out.get())();
     cl_command_queue queue = getQueue()();
 
-    CLFFT_CHECK(clfftEnqueueTransform(plan,
+    CLFFT_CHECK(clfftEnqueueTransform(*plan.get(),
                                       CLFFT_BACKWARD,
                                       1, &queue, 0, NULL, NULL,
                                       &imem, &omem, NULL));
diff --git a/src/backend/opencl/hist_graphics.cpp b/src/backend/opencl/hist_graphics.cpp
index 701c8c337a..798363769e 100644
--- a/src/backend/opencl/hist_graphics.cpp
+++ b/src/backend/opencl/hist_graphics.cpp
@@ -27,12 +27,10 @@ void copy_histogram(const Array<T> &data, const forge::Histogram* hist)
         const cl::Buffer *d_P = data.get();
         size_t bytes = hist->verticesSize();
 
-        GraphicsResourceManager& intrpMngr = interopManager();
-
-        cl::Buffer **resources = intrpMngr.getBufferResource(hist);
+        ShrdResVector res = interopManager().getBufferResource(hist);
 
         std::vector<cl::Memory> shared_objects;
-        shared_objects.push_back(*resources[0]);
+        shared_objects.push_back(*(res[0].get()));
 
         glFinish();
 
@@ -42,7 +40,7 @@ void copy_histogram(const Array<T> &data, const forge::Histogram* hist)
 
         getQueue().enqueueAcquireGLObjects(&shared_objects, NULL, &event);
         event.wait();
-        getQueue().enqueueCopyBuffer(*d_P, *resources[0], 0, 0, bytes, NULL, &event);
+        getQueue().enqueueCopyBuffer(*d_P, *(res[0].get()), 0, 0, bytes, NULL, &event);
         getQueue().enqueueReleaseGLObjects(&shared_objects, NULL, &event);
         event.wait();
 
diff --git a/src/backend/opencl/image.cpp b/src/backend/opencl/image.cpp
index 4df7f5cde5..7acf5a66c9 100644
--- a/src/backend/opencl/image.cpp
+++ b/src/backend/opencl/image.cpp
@@ -27,14 +27,14 @@ void copy_image(const Array<T> &in, const forge::Image* image)
 {
     if (isGLSharingSupported()) {
         CheckGL("Begin opencl resource copy");
-        GraphicsResourceManager& intrpMngr = interopManager();
 
-        cl::Buffer **resources = intrpMngr.getBufferResource(image);
+        ShrdResVector res = interopManager().getBufferResource(image);
+
         const cl::Buffer *d_X = in.get();
         size_t num_bytes = image->size();
 
         std::vector<cl::Memory> shared_objects;
-        shared_objects.push_back(*resources[0]);
+        shared_objects.push_back(*(res[0].get()));
 
         glFinish();
 
@@ -44,7 +44,7 @@ void copy_image(const Array<T> &in, const forge::Image* image)
 
         getQueue().enqueueAcquireGLObjects(&shared_objects, NULL, &event);
         event.wait();
-        getQueue().enqueueCopyBuffer(*d_X, *resources[0], 0, 0, num_bytes, NULL, &event);
+        getQueue().enqueueCopyBuffer(*d_X, *(res[0].get()), 0, 0, num_bytes, NULL, &event);
         getQueue().enqueueReleaseGLObjects(&shared_objects, NULL, &event);
         event.wait();
 
diff --git a/src/backend/opencl/platform.cpp b/src/backend/opencl/platform.cpp
index 910d500a58..1f11a54937 100644
--- a/src/backend/opencl/platform.cpp
+++ b/src/backend/opencl/platform.cpp
@@ -624,7 +624,15 @@ GraphicsResourceManager& interopManager()
 
 PlanCache& fftManager()
 {
-    return DeviceManager::getInstance().clfftManagers[getActiveDeviceId()];
+    static std::once_flag initFlags[DeviceManager::MAX_DEVICES];
+
+    int id = getActiveDeviceId();
+
+    DeviceManager& inst = DeviceManager::getInstance();
+
+    std::call_once(initFlags[id], [&]{ inst.clfftManagers[id].reset(new PlanCache()); });
+
+    return *(inst.clfftManagers[id].get());
 }
 
 DeviceManager& DeviceManager::getInstance()
@@ -635,8 +643,16 @@ DeviceManager& DeviceManager::getInstance()
 
 DeviceManager::~DeviceManager()
 {
-    for (int i=0; i<getDeviceCount(); ++i)
+    for (int i=0; i<getDeviceCount(); ++i) {
         delete gfxManagers[i].release();
+        delete clfftManagers[i].release();
+    }
+#ifndef OS_WIN
+    //TODO: FIXME:
+    // clfftTeardown() causes a "Pure Virtual Function Called" crash on
+    // Windows for Intel devices. This causes tests to fail.
+    clfftTeardown();
+#endif
     delete memManager.release();
     delete pinnedMemManager.release();
 
@@ -649,8 +665,8 @@ DeviceManager::~DeviceManager()
     // So, clean up OpenCL resources on non-Windows platforms
 #ifndef OS_WIN
     for (auto q: mQueues) delete q;
-    for (auto d : mDevices) delete d;
-    for (auto c : mContexts) delete c;
+    for (auto c: mContexts) delete c;
+    for (auto d: mDevices) delete d;
 #endif
 }
 
@@ -784,6 +800,9 @@ DeviceManager::DeviceManager()
     }
 #endif
     mUserDeviceOffset = mDevices.size();
+    //Initialize FFT setup data structure
+    CLFFT_CHECK(clfftInitSetupData(&mFFTSetup));
+    CLFFT_CHECK(clfftSetup(&mFFTSetup));
 }
 
 #if defined(WITH_GRAPHICS)
diff --git a/src/backend/opencl/platform.hpp b/src/backend/opencl/platform.hpp
index 846b839ee5..fe9ded084f 100644
--- a/src/backend/opencl/platform.hpp
+++ b/src/backend/opencl/platform.hpp
@@ -180,6 +180,7 @@ class DeviceManager
         std::unique_ptr<MemoryManagerPinned> pinnedMemManager;
 
         std::unique_ptr<GraphicsResourceManager> gfxManagers[MAX_DEVICES];
-        PlanCache clfftManagers[MAX_DEVICES];
+        clfftSetupData mFFTSetup;
+        std::unique_ptr<PlanCache> clfftManagers[MAX_DEVICES];
 };
 }
diff --git a/src/backend/opencl/plot.cpp b/src/backend/opencl/plot.cpp
index cd8134fe4f..4651d60737 100644
--- a/src/backend/opencl/plot.cpp
+++ b/src/backend/opencl/plot.cpp
@@ -32,12 +32,10 @@ void copy_plot(const Array<T> &P, forge::Plot* plot)
         const cl::Buffer *d_P = P.get();
         size_t bytes = plot->verticesSize();
 
-        GraphicsResourceManager& intrpMngr = interopManager();
-
-        cl::Buffer **resources = intrpMngr.getBufferResource(plot);
+        ShrdResVector res = interopManager().getBufferResource(plot);
 
         std::vector<cl::Memory> shared_objects;
-        shared_objects.push_back(*resources[0]);
+        shared_objects.push_back(*(res[0].get()));
 
         glFinish();
 
@@ -47,7 +45,7 @@ void copy_plot(const Array<T> &P, forge::Plot* plot)
 
         getQueue().enqueueAcquireGLObjects(&shared_objects, NULL, &event);
         event.wait();
-        getQueue().enqueueCopyBuffer(*d_P, *resources[0], 0, 0, bytes, NULL, &event);
+        getQueue().enqueueCopyBuffer(*d_P, *(res[0].get()), 0, 0, bytes, NULL, &event);
         getQueue().enqueueReleaseGLObjects(&shared_objects, NULL, &event);
         event.wait();
 
diff --git a/src/backend/opencl/surface.cpp b/src/backend/opencl/surface.cpp
index ad296dbb54..ef7782c561 100644
--- a/src/backend/opencl/surface.cpp
+++ b/src/backend/opencl/surface.cpp
@@ -32,12 +32,10 @@ void copy_surface(const Array<T> &P, forge::Surface* surface)
         const cl::Buffer *d_P = P.get();
         size_t bytes = surface->verticesSize();
 
-        GraphicsResourceManager& intrpMngr = interopManager();
-
-        cl::Buffer **resources = intrpMngr.getBufferResource(surface);
+        ShrdResVector res = interopManager().getBufferResource(surface);
 
         std::vector<cl::Memory> shared_objects;
-        shared_objects.push_back(*resources[0]);
+        shared_objects.push_back(*(res[0].get()));
 
         glFinish();
 
@@ -47,7 +45,7 @@ void copy_surface(const Array<T> &P, forge::Surface* surface)
 
         getQueue().enqueueAcquireGLObjects(&shared_objects, NULL, &event);
         event.wait();
-        getQueue().enqueueCopyBuffer(*d_P, *resources[0], 0, 0, bytes, NULL, &event);
+        getQueue().enqueueCopyBuffer(*d_P, *(res[0].get()), 0, 0, bytes, NULL, &event);
         getQueue().enqueueReleaseGLObjects(&shared_objects, NULL, &event);
         event.wait();
 
diff --git a/src/backend/opencl/vector_field.cpp b/src/backend/opencl/vector_field.cpp
index 53ac16d88a..c53279aa0c 100644
--- a/src/backend/opencl/vector_field.cpp
+++ b/src/backend/opencl/vector_field.cpp
@@ -32,13 +32,11 @@ void copy_vector_field(const Array<T> &points, const Array<T> &directions,
         size_t pBytes = vector_field->verticesSize();
         size_t dBytes = vector_field->directionsSize();
 
-        GraphicsResourceManager& intrpMngr = interopManager();
-
-        cl::Buffer **resources = intrpMngr.getBufferResource(vector_field);
+        ShrdResVector res = interopManager().getBufferResource(vector_field);
 
         std::vector<cl::Memory> shared_objects;
-        shared_objects.push_back(*resources[0]);
-        shared_objects.push_back(*resources[1]);
+        shared_objects.push_back(*(res[0].get()));
+        shared_objects.push_back(*(res[1].get()));
 
         glFinish();
 
@@ -48,8 +46,8 @@ void copy_vector_field(const Array<T> &points, const Array<T> &directions,
 
         getQueue().enqueueAcquireGLObjects(&shared_objects, NULL, &event);
         event.wait();
-        getQueue().enqueueCopyBuffer(*d_points    , *resources[0], 0, 0, pBytes, NULL, &event);
-        getQueue().enqueueCopyBuffer(*d_directions, *resources[1], 0, 0, dBytes, NULL, &event);
+        getQueue().enqueueCopyBuffer(*d_points    , *(res[0].get()), 0, 0, pBytes, NULL, &event);
+        getQueue().enqueueCopyBuffer(*d_directions, *(res[1].get()), 0, 0, dBytes, NULL, &event);
         getQueue().enqueueReleaseGLObjects(&shared_objects, NULL, &event);
         event.wait();
 

From 73ebc03a38866d1a8a60380aeda90e530aba3bad Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Mon, 30 Jan 2017 21:18:01 +0530
Subject: [PATCH 32/82] Fix cmake statement in glbinding build script

---
 CMakeModules/build_glbinding.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeModules/build_glbinding.cmake b/CMakeModules/build_glbinding.cmake
index c929778d92..3f4f493dc7 100644
--- a/CMakeModules/build_glbinding.cmake
+++ b/CMakeModules/build_glbinding.cmake
@@ -3,7 +3,7 @@ INCLUDE(ExternalProject)
 SET(prefix ${PROJECT_BINARY_DIR}/third_party/glb)
 
 SET(LIB_POSTFIX "")
-IF (${CMAKE_BUILD_TYPE} STREQUAL "Debug")
+IF (${CMAKE_BUILD_TYPE} MATCHES DEBUG)
     SET(LIB_POSTFIX "d")
 ENDIF()
 

From bcd03ef0743d6191a2b386c540a6c3adb965407b Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Mon, 30 Jan 2017 21:18:43 +0530
Subject: [PATCH 33/82] tyepdef typo fix in common::FFTPlanCache class

---
 src/backend/common/FFTPlanCache.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/backend/common/FFTPlanCache.cpp b/src/backend/common/FFTPlanCache.cpp
index 0360cb943c..be5f4d9582 100644
--- a/src/backend/common/FFTPlanCache.cpp
+++ b/src/backend/common/FFTPlanCache.cpp
@@ -43,7 +43,7 @@ std::shared_ptr<P> FFTPlanCache<T, P>::find(const std::string& key) const
     std::shared_ptr<P> res;
 
     rlock_t lock(gFFTMutexes[detail::getActiveDeviceId()]);
-    for(uint i=0; i<mCache.size(); ++i) {
+    for(unsigned i=0; i<mCache.size(); ++i) {
         if (key == mCache[i].first) {
             res = mCache[i].second;
             break;

From 76cde992bcdcd52cf9a49e2d8c0186d7e5cdf93e Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Wed, 1 Feb 2017 23:24:51 +0530
Subject: [PATCH 34/82] Revert "Fix cmake statement in glbinding build script"

This reverts commit 73ebc03a38866d1a8a60380aeda90e530aba3bad.

Reverted change fixed build configuration generation error on Windows
platform but the condition is not being evaluated correctly on non-single
configuration generators such as make and ninja.
---
 CMakeModules/build_glbinding.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeModules/build_glbinding.cmake b/CMakeModules/build_glbinding.cmake
index 3f4f493dc7..c929778d92 100644
--- a/CMakeModules/build_glbinding.cmake
+++ b/CMakeModules/build_glbinding.cmake
@@ -3,7 +3,7 @@ INCLUDE(ExternalProject)
 SET(prefix ${PROJECT_BINARY_DIR}/third_party/glb)
 
 SET(LIB_POSTFIX "")
-IF (${CMAKE_BUILD_TYPE} MATCHES DEBUG)
+IF (${CMAKE_BUILD_TYPE} STREQUAL "Debug")
     SET(LIB_POSTFIX "d")
 ENDIF()
 

From bb4e2823274dab7fbc56a5b9b8e2c8d87cbf5590 Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Thu, 2 Feb 2017 09:47:14 +0530
Subject: [PATCH 35/82] fix build type check in glbinding cmake build script

---
 CMakeModules/build_glbinding.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeModules/build_glbinding.cmake b/CMakeModules/build_glbinding.cmake
index c929778d92..05ad586e76 100644
--- a/CMakeModules/build_glbinding.cmake
+++ b/CMakeModules/build_glbinding.cmake
@@ -3,7 +3,7 @@ INCLUDE(ExternalProject)
 SET(prefix ${PROJECT_BINARY_DIR}/third_party/glb)
 
 SET(LIB_POSTFIX "")
-IF (${CMAKE_BUILD_TYPE} STREQUAL "Debug")
+IF (CMAKE_BUILD_TYPE MATCHES Debug)
     SET(LIB_POSTFIX "d")
 ENDIF()
 

From afda285603ca1da2a354c90205404cf5b545f279 Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Thu, 2 Feb 2017 15:26:27 +0530
Subject: [PATCH 36/82] change fft manager cache managers to be thread local

Though the upstream libraries(clFFT/cuFFT) can guarantee thread safety,
they can do so only if same fft plan is not being used from two different
threads. Hence, it is safe to cache fft plans on per thread basis.
---
 src/backend/cuda/platform.cpp   |  4 +++-
 src/backend/cuda/platform.hpp   |  2 --
 src/backend/opencl/platform.cpp | 11 ++---------
 src/backend/opencl/platform.hpp |  1 -
 4 files changed, 5 insertions(+), 13 deletions(-)

diff --git a/src/backend/cuda/platform.cpp b/src/backend/cuda/platform.cpp
index ce55bca8eb..847be47c3c 100644
--- a/src/backend/cuda/platform.cpp
+++ b/src/backend/cuda/platform.cpp
@@ -421,7 +421,9 @@ GraphicsResourceManager& interopManager()
 
 PlanCache& fftManager()
 {
-    return DeviceManager::getInstance().cufftManagers[getActiveDeviceId()];
+    thread_local static PlanCache cufftManagers[DeviceManager::MAX_DEVICES];
+
+    return cufftManagers[getActiveDeviceId()];
 }
 
 BlasHandle blasHandle()
diff --git a/src/backend/cuda/platform.hpp b/src/backend/cuda/platform.hpp
index ff2a9f8a79..f91e9da641 100644
--- a/src/backend/cuda/platform.hpp
+++ b/src/backend/cuda/platform.hpp
@@ -167,8 +167,6 @@ class DeviceManager
 
         std::unique_ptr<GraphicsResourceManager> gfxManagers[MAX_DEVICES];
 
-        PlanCache cufftManagers[MAX_DEVICES];
-
         std::unique_ptr<cublasHandle> cublasHandles[MAX_DEVICES];
 
         std::unique_ptr<cusolverDnHandle> cusolverHandles[MAX_DEVICES];
diff --git a/src/backend/opencl/platform.cpp b/src/backend/opencl/platform.cpp
index 1f11a54937..67dd594e08 100644
--- a/src/backend/opencl/platform.cpp
+++ b/src/backend/opencl/platform.cpp
@@ -624,15 +624,9 @@ GraphicsResourceManager& interopManager()
 
 PlanCache& fftManager()
 {
-    static std::once_flag initFlags[DeviceManager::MAX_DEVICES];
-
-    int id = getActiveDeviceId();
-
-    DeviceManager& inst = DeviceManager::getInstance();
-
-    std::call_once(initFlags[id], [&]{ inst.clfftManagers[id].reset(new PlanCache()); });
+    thread_local static PlanCache clfftManagers[DeviceManager::MAX_DEVICES];
 
-    return *(inst.clfftManagers[id].get());
+    return clfftManagers[getActiveDeviceId()];
 }
 
 DeviceManager& DeviceManager::getInstance()
@@ -645,7 +639,6 @@ DeviceManager::~DeviceManager()
 {
     for (int i=0; i<getDeviceCount(); ++i) {
         delete gfxManagers[i].release();
-        delete clfftManagers[i].release();
     }
 #ifndef OS_WIN
     //TODO: FIXME:
diff --git a/src/backend/opencl/platform.hpp b/src/backend/opencl/platform.hpp
index fe9ded084f..66a1acb53a 100644
--- a/src/backend/opencl/platform.hpp
+++ b/src/backend/opencl/platform.hpp
@@ -181,6 +181,5 @@ class DeviceManager
 
         std::unique_ptr<GraphicsResourceManager> gfxManagers[MAX_DEVICES];
         clfftSetupData mFFTSetup;
-        std::unique_ptr<PlanCache> clfftManagers[MAX_DEVICES];
 };
 }

From 60df0089fc6ef7c03cb5b880930c62e698dd9540 Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Mon, 20 Feb 2017 22:17:31 +0530
Subject: [PATCH 37/82] Make opencl kernel cache management thread safe

---
 src/backend/opencl/cache.hpp                  | 10 +--
 src/backend/opencl/jit.cpp                    | 13 ++--
 .../opencl/kernel/convolve/conv2_impl.hpp     | 10 ++-
 .../opencl/kernel/convolve_separable.cpp      | 11 ++-
 src/backend/opencl/kernel/cscmm.hpp           | 10 +--
 src/backend/opencl/kernel/cscmv.hpp           | 10 +--
 src/backend/opencl/kernel/csrmm.hpp           | 10 +--
 src/backend/opencl/kernel/csrmv.hpp           | 10 +--
 src/backend/opencl/kernel/fast.hpp            | 38 +++++-----
 src/backend/opencl/kernel/ireduce.hpp         | 60 ++++++++--------
 src/backend/opencl/kernel/moments.hpp         |  9 +--
 .../opencl/kernel/nearest_neighbour.hpp       | 72 +++++++++----------
 src/backend/opencl/kernel/random_engine.hpp   | 24 +++----
 src/backend/opencl/kernel/reduce.hpp          | 19 ++---
 src/backend/opencl/kernel/scan_dim.hpp        |  9 +--
 .../opencl/kernel/scan_dim_by_key_impl.hpp    | 11 ++-
 src/backend/opencl/kernel/scan_first.hpp      | 10 +--
 .../opencl/kernel/scan_first_by_key_impl.hpp  | 10 +--
 src/backend/opencl/kernel/sparse.hpp          | 54 ++++++--------
 src/backend/opencl/kernel/transform.hpp       |  9 ++-
 src/backend/opencl/kernel/unwrap.hpp          |  9 +--
 src/backend/opencl/kernel/wrap.hpp            |  9 +--
 src/backend/opencl/platform.cpp               | 36 ++++++++++
 src/backend/opencl/platform.hpp               | 15 ++++
 24 files changed, 237 insertions(+), 241 deletions(-)

diff --git a/src/backend/opencl/cache.hpp b/src/backend/opencl/cache.hpp
index 937d3d8eac..0dab9d9e24 100644
--- a/src/backend/opencl/cache.hpp
+++ b/src/backend/opencl/cache.hpp
@@ -14,14 +14,10 @@
 
 namespace opencl
 {
-    using cl::Kernel;
-    using cl::Program;
-
     typedef struct {
-        Program* prog;
-        Kernel* ker;
+        cl::Program* prog;
+        cl::Kernel* ker;
     } kc_entry_t;
 
-    typedef std::map<string, kc_entry_t> kc_t;
-    static kc_t kernelCaches[DeviceManager::MAX_DEVICES];
+    typedef std::map<std::string, kc_entry_t> kc_t;
 }
diff --git a/src/backend/opencl/jit.cpp b/src/backend/opencl/jit.cpp
index 83aab34186..360028d967 100644
--- a/src/backend/opencl/jit.cpp
+++ b/src/backend/opencl/jit.cpp
@@ -16,7 +16,6 @@
 #include <JIT/Node.hpp>
 #include <kernel_headers/jit.hpp>
 #include <program.hpp>
-#include <cache.hpp>
 #include <dispatch.hpp>
 #include <err_opencl.hpp>
 #include <functional>
@@ -156,27 +155,25 @@ static string getKernelString(string funcName, std::vector<Node *> nodes, bool i
 
 static Kernel getKernel(std::vector<Node *> nodes, bool is_linear)
 {
-
     bool is_dbl = false;
     string funcName = getFuncName(nodes, is_linear, &is_dbl);
     int device = getActiveDeviceId();
 
-    kc_t::iterator idx = kernelCaches[device].find(funcName);
-    kc_entry_t entry;
+    kc_entry_t entry = kernelCache(device, funcName);
 
-    if (idx == kernelCaches[device].end()) {
+    if (entry.prog==0 && entry.ker==0) {
         string jit_ker = getKernelString(funcName, nodes, is_linear);
 
         const char *ker_strs[] = {jit_cl, jit_ker.c_str()};
         const int ker_lens[] = {jit_cl_len, (int)jit_ker.size()};
+
         cl::Program prog;
         buildProgram(prog, 2, ker_strs, ker_lens, is_dbl ? string(" -D USE_DOUBLE") :  string(""));
+
         entry.prog = new cl::Program(prog);
         entry.ker = new Kernel(*entry.prog, funcName.c_str());
 
-        kernelCaches[device][funcName] = entry;
-    } else {
-        entry = idx->second;
+        addKernelToCache(device, funcName, entry);
     }
 
     return *entry.ker;
diff --git a/src/backend/opencl/kernel/convolve/conv2_impl.hpp b/src/backend/opencl/kernel/convolve/conv2_impl.hpp
index 04fae02d72..ef8df00d3b 100644
--- a/src/backend/opencl/kernel/convolve/conv2_impl.hpp
+++ b/src/backend/opencl/kernel/convolve/conv2_impl.hpp
@@ -35,10 +35,10 @@ void conv2Helper(const conv_kparam_t& param, Param out, const Param signal, cons
         std::to_string(f1);
 
     int device = getActiveDeviceId();
-    kc_t::iterator idx = kernelCaches[device].find(ref_name);
 
-    kc_entry_t entry;
-    if (idx == kernelCaches[device].end()) {
+    kc_entry_t entry = kernelCache(device, ref_name);
+
+    if (entry.prog==0 && entry.ker==0) {
         size_t LOC_SIZE = (THREADS_X+2*(f0-1))*(THREADS_Y+2*(f1-1));
 
         std::ostringstream options;
@@ -58,9 +58,7 @@ void conv2Helper(const conv_kparam_t& param, Param out, const Param signal, cons
         entry.prog   = new Program(prog);
         entry.ker = new Kernel(*entry.prog, "convolve");
 
-        kernelCaches[device][ref_name] = entry;
-    } else {
-        entry = idx->second;
+        addKernelToCache(device, ref_name, entry);
     }
 
     auto convOp = cl::KernelFunctor<Buffer, KParam, Buffer, KParam,
diff --git a/src/backend/opencl/kernel/convolve_separable.cpp b/src/backend/opencl/kernel/convolve_separable.cpp
index b212cbe452..aafc4253ef 100644
--- a/src/backend/opencl/kernel/convolve_separable.cpp
+++ b/src/backend/opencl/kernel/convolve_separable.cpp
@@ -54,10 +54,10 @@ void convSep(Param out, const Param signal, const Param filter)
         std::to_string(fLen);
 
     int device = getActiveDeviceId();
-    kc_t::iterator idx = kernelCaches[device].find(ref_name);
 
-    kc_entry_t entry;
-    if (idx == kernelCaches[device].end()) {
+    kc_entry_t entry = kernelCache(device, ref_name);
+
+    if (entry.prog==0 && entry.ker==0) {
         const size_t C0_SIZE  = (THREADS_X+2*(fLen-1))* THREADS_Y;
         const size_t C1_SIZE  = (THREADS_Y+2*(fLen-1))* THREADS_X;
 
@@ -79,9 +79,8 @@ void convSep(Param out, const Param signal, const Param filter)
 
         entry.prog   = new Program(prog);
         entry.ker  = new Kernel(*entry.prog, "convolve");
-        kernelCaches[device][ref_name] = entry;
-    } else {
-        entry = idx->second;
+
+        addKernelToCache(device, ref_name, entry);
     }
 
     auto convOp = KernelFunctor<Buffer, KParam, Buffer, KParam, Buffer,
diff --git a/src/backend/opencl/kernel/cscmm.hpp b/src/backend/opencl/kernel/cscmm.hpp
index 3b6bc13827..a72a150501 100644
--- a/src/backend/opencl/kernel/cscmm.hpp
+++ b/src/backend/opencl/kernel/cscmm.hpp
@@ -68,10 +68,10 @@ namespace opencl
                 std::to_string(threads);
 
             int device = getActiveDeviceId();
-            auto idx = kernelCaches[device].find(ref_name);
-            kc_entry_t entry;
 
-            if (idx == kernelCaches[device].end()) {
+            kc_entry_t entry = kernelCache(device, ref_name);
+
+            if (entry.prog==0 && entry.ker==0) {
 
                 std::ostringstream options;
                 options << " -D T=" << dtype_traits<T>::getName();
@@ -100,8 +100,8 @@ namespace opencl
                 buildProgram(prog, 1, ker_strs, ker_lens, options.str());
                 entry.prog = new Program(prog);
                 entry.ker  = new Kernel(*entry.prog, "cscmm_nn");
-            } else {
-                entry = idx->second;
+
+                addKernelToCache(device, ref_name, entry);
             }
 
             auto cscmm_kernel = *entry.ker;
diff --git a/src/backend/opencl/kernel/cscmv.hpp b/src/backend/opencl/kernel/cscmv.hpp
index 6aabe2cfd3..2a2b676c44 100644
--- a/src/backend/opencl/kernel/cscmv.hpp
+++ b/src/backend/opencl/kernel/cscmv.hpp
@@ -65,10 +65,10 @@ namespace opencl
                 std::to_string(threads);
 
             int device = getActiveDeviceId();
-            auto idx = kernelCaches[device].find(ref_name);
-            kc_entry_t entry;
 
-            if (idx == kernelCaches[device].end()) {
+            kc_entry_t entry = kernelCache(device, ref_name);
+
+            if (entry.prog==0 && entry.ker==0) {
 
                 std::ostringstream options;
                 options << " -D T=" << dtype_traits<T>::getName();
@@ -96,8 +96,8 @@ namespace opencl
                 buildProgram(prog, 1, ker_strs, ker_lens, options.str());
                 entry.prog = new Program(prog);
                 entry.ker  = new Kernel(*entry.prog, "cscmv_block");
-            } else {
-                entry = idx->second;
+
+                addKernelToCache(device, ref_name, entry);
             }
 
             auto cscmv_kernel = *entry.ker;
diff --git a/src/backend/opencl/kernel/csrmm.hpp b/src/backend/opencl/kernel/csrmm.hpp
index 0afab2973c..40e2bd30e6 100644
--- a/src/backend/opencl/kernel/csrmm.hpp
+++ b/src/backend/opencl/kernel/csrmm.hpp
@@ -61,10 +61,10 @@ namespace opencl
                 std::to_string(use_greedy);
 
             int device = getActiveDeviceId();
-            auto idx = kernelCaches[device].find(ref_name);
-            kc_entry_t entry;
 
-            if (idx == kernelCaches[device].end()) {
+            kc_entry_t entry = kernelCache(device, ref_name);
+
+            if (entry.prog==0 && entry.ker==0) {
 
                 std::ostringstream options;
                 options << " -D T=" << dtype_traits<T>::getName();
@@ -94,8 +94,8 @@ namespace opencl
                 entry.ker[0] = Kernel(*entry.prog, "csrmm_nt");
                 // FIXME: Change this after adding another kernel
                 entry.ker[1] = Kernel(*entry.prog, "csrmm_nt");
-            } else {
-                entry = idx->second;
+
+                addKernelToCache(device, ref_name, entry);
             }
 
             auto csrmm_nt_kernel = entry.ker[0];
diff --git a/src/backend/opencl/kernel/csrmv.hpp b/src/backend/opencl/kernel/csrmv.hpp
index faffb828e8..2ebb0b964e 100644
--- a/src/backend/opencl/kernel/csrmv.hpp
+++ b/src/backend/opencl/kernel/csrmv.hpp
@@ -67,10 +67,10 @@ namespace opencl
                 std::to_string(threads);
 
             int device = getActiveDeviceId();
-            auto idx = kernelCaches[device].find(ref_name);
-            kc_entry_t entry;
 
-            if (idx == kernelCaches[device].end()) {
+            kc_entry_t entry = kernelCache(device, ref_name);
+
+            if (entry.prog==0 && entry.ker==0) {
 
                 std::ostringstream options;
                 options << " -D T=" << dtype_traits<T>::getName();
@@ -99,8 +99,8 @@ namespace opencl
                 entry.ker  = new Kernel[2];
                 entry.ker[0] = Kernel(*entry.prog, "csrmv_thread");
                 entry.ker[1] = Kernel(*entry.prog, "csrmv_block");
-            } else {
-                entry = idx->second;
+
+                addKernelToCache(device, ref_name, entry);
             }
 
             int count = 0;
diff --git a/src/backend/opencl/kernel/fast.hpp b/src/backend/opencl/kernel/fast.hpp
index ec66867b93..bf719e810d 100644
--- a/src/backend/opencl/kernel/fast.hpp
+++ b/src/backend/opencl/kernel/fast.hpp
@@ -56,33 +56,31 @@ void fast(const unsigned arc_length,
         std::string(dtype_traits<T>::getName());
 
     int device = getActiveDeviceId();
-    kc_t::iterator cache_idx = kernelCaches[device].find(ref_name);
 
-    kc_entry_t entry;
-    if (cache_idx == kernelCaches[device].end()) {
+    kc_entry_t entry = kernelCache(device, ref_name);
 
-            std::ostringstream options;
-            options << " -D T=" << dtype_traits<T>::getName()
-                    << " -D ARC_LENGTH=" << arc_length
-                    << " -D NONMAX=" << static_cast<unsigned>(nonmax);
+    if (entry.prog==0 && entry.ker==0) {
 
-            if (std::is_same<T, double>::value ||
+        std::ostringstream options;
+        options << " -D T=" << dtype_traits<T>::getName()
+            << " -D ARC_LENGTH=" << arc_length
+            << " -D NONMAX=" << static_cast<unsigned>(nonmax);
+
+        if (std::is_same<T, double>::value ||
                 std::is_same<T, cdouble>::value) {
-                options << " -D USE_DOUBLE";
-            }
+            options << " -D USE_DOUBLE";
+        }
 
-            cl::Program prog;
-            buildProgram(prog, fast_cl, fast_cl_len, options.str());
-            entry.prog = new Program(prog);
-            entry.ker = new Kernel[3];
+        cl::Program prog;
+        buildProgram(prog, fast_cl, fast_cl_len, options.str());
+        entry.prog = new Program(prog);
+        entry.ker = new Kernel[3];
 
-            entry.ker[0] = Kernel(*entry.prog, "locate_features");
-            entry.ker[1] = Kernel(*entry.prog, "non_max_counts");
-            entry.ker[2] = Kernel(*entry.prog, "get_features");
+        entry.ker[0] = Kernel(*entry.prog, "locate_features");
+        entry.ker[1] = Kernel(*entry.prog, "non_max_counts");
+        entry.ker[2] = Kernel(*entry.prog, "get_features");
 
-            kernelCaches[device][ref_name] = entry;
-    } else {
-        entry = cache_idx -> second;
+        addKernelToCache(device, ref_name, entry);
     }
 
     const unsigned max_feat = ceil(in.info.dims[0] * in.info.dims[1] * feature_ratio);
diff --git a/src/backend/opencl/kernel/ireduce.hpp b/src/backend/opencl/kernel/ireduce.hpp
index 2c76f2720f..ab71beb6df 100644
--- a/src/backend/opencl/kernel/ireduce.hpp
+++ b/src/backend/opencl/kernel/ireduce.hpp
@@ -62,39 +62,37 @@ namespace kernel
             std::to_string(threads_y);
 
         int device = getActiveDeviceId();
-        kc_t::iterator idx = kernelCaches[device].find(ref_name);
 
-        kc_entry_t entry;
-        if (idx == kernelCaches[device].end()) {
+        kc_entry_t entry = kernelCache(device, ref_name);
 
-                Binary<T, op> ireduce;
-                ToNumStr<T> toNumStr;
+        if (entry.prog==0 && entry.ker==0) {
 
-                std::ostringstream options;
-                options << " -D T=" << dtype_traits<T>::getName()
-                        << " -D dim=" << dim
-                        << " -D DIMY=" << threads_y
-                        << " -D THREADS_X=" << THREADS_X
-                        << " -D init=" << toNumStr(ireduce.init())
-                        << " -D " << binOpName<op>()
-                        << " -D CPLX=" << af::iscplx<T>()
-                        << " -D IS_FIRST=" << is_first;
+            Binary<T, op> ireduce;
+            ToNumStr<T> toNumStr;
+
+            std::ostringstream options;
+            options << " -D T=" << dtype_traits<T>::getName()
+                << " -D dim=" << dim
+                << " -D DIMY=" << threads_y
+                << " -D THREADS_X=" << THREADS_X
+                << " -D init=" << toNumStr(ireduce.init())
+                << " -D " << binOpName<op>()
+                << " -D CPLX=" << af::iscplx<T>()
+                << " -D IS_FIRST=" << is_first;
 
-                if (std::is_same<T, double>::value ||
+            if (std::is_same<T, double>::value ||
                     std::is_same<T, cdouble>::value) {
-                    options << " -D USE_DOUBLE";
-                }
+                options << " -D USE_DOUBLE";
+            }
 
-                const char *ker_strs[] = {iops_cl, ireduce_dim_cl};
-                const int   ker_lens[] = {iops_cl_len, ireduce_dim_cl_len};
-                Program prog;
-                buildProgram(prog, 2, ker_strs, ker_lens, options.str());
-                entry.prog = new Program(prog);
-                entry.ker = new Kernel(*entry.prog, "ireduce_dim_kernel");
+            const char *ker_strs[] = {iops_cl, ireduce_dim_cl};
+            const int   ker_lens[] = {iops_cl_len, ireduce_dim_cl_len};
+            Program prog;
+            buildProgram(prog, 2, ker_strs, ker_lens, options.str());
+            entry.prog = new Program(prog);
+            entry.ker = new Kernel(*entry.prog, "ireduce_dim_kernel");
 
-                kernelCaches[device][ref_name] = entry;
-        } else {
-            entry = idx->second;
+            addKernelToCache(device, ref_name, entry);
         }
 
         NDRange local(THREADS_X, threads_y);
@@ -174,10 +172,10 @@ namespace kernel
             std::to_string(threads_x);
 
         int device = getActiveDeviceId();
-        kc_t::iterator idx = kernelCaches[device].find(ref_name);
 
-        kc_entry_t entry;
-        if (idx == kernelCaches[device].end()) {
+        kc_entry_t entry = kernelCache(device, ref_name);
+
+        if (entry.prog==0 && entry.ker==0) {
 
             Binary<T, op> ireduce;
             ToNumStr<T> toNumStr;
@@ -203,9 +201,7 @@ namespace kernel
             entry.prog = new Program(prog);
             entry.ker = new Kernel(*entry.prog, "ireduce_first_kernel");
 
-            kernelCaches[device][ref_name] = entry;
-        } else {
-            entry = idx->second;
+            addKernelToCache(device, ref_name, entry);
         }
 
         NDRange local(threads_x, THREADS_PER_GROUP / threads_x);
diff --git a/src/backend/opencl/kernel/moments.hpp b/src/backend/opencl/kernel/moments.hpp
index 10200e2aa7..a4fafe2b5e 100644
--- a/src/backend/opencl/kernel/moments.hpp
+++ b/src/backend/opencl/kernel/moments.hpp
@@ -49,10 +49,9 @@ namespace opencl
             std::to_string(out.info.dims[0]);
 
             int device = getActiveDeviceId();
-            kc_t::iterator idx = kernelCaches[device].find(ref_name);
+            kc_entry_t entry = kernelCache(device, ref_name);
 
-            kc_entry_t entry;
-            if (idx == kernelCaches[device].end()) {
+            if (entry.prog==0 && entry.ker==0) {
                 std::ostringstream options;
                 options << " -D T="        << dtype_traits<T>::getName();
                 options << " -D MOMENTS_SZ=" << out.info.dims[0];
@@ -69,9 +68,7 @@ namespace opencl
                 entry.prog = new Program(prog);
                 entry.ker = new Kernel(*entry.prog, "moments_kernel");
 
-                kernelCaches[device][ref_name] = entry;
-            } else {
-                entry = idx->second;
+                addKernelToCache(device, ref_name, entry);
             }
 
 
diff --git a/src/backend/opencl/kernel/nearest_neighbour.hpp b/src/backend/opencl/kernel/nearest_neighbour.hpp
index 328708cfe9..7280508a63 100644
--- a/src/backend/opencl/kernel/nearest_neighbour.hpp
+++ b/src/backend/opencl/kernel/nearest_neighbour.hpp
@@ -61,49 +61,47 @@ void nearest_neighbour(Param idx,
         std::to_string(unroll_len);
 
     int device = getActiveDeviceId();
-    kc_t::iterator cache_idx = kernelCaches[device].find(ref_name);
-
-    kc_entry_t entry;
-    if (cache_idx == kernelCaches[device].end()) {
-
-            std::ostringstream options;
-            options << " -D T=" << dtype_traits<T>::getName()
-                    << " -D To=" << dtype_traits<To>::getName()
-                    << " -D THREADS=" << THREADS
-                    << " -D FEAT_LEN=" << unroll_len;
-
-            switch(dist_type) {
-                case AF_SAD: options <<" -D DISTOP=_sad_"; break;
-                case AF_SSD: options <<" -D DISTOP=_ssd_"; break;
-                case AF_SHD: options <<" -D DISTOP=_shd_ -D __SHD__";
-                              break;
-                default: break;
-            }
-
-            if (std::is_same<T, double>::value ||
+
+    kc_entry_t entry = kernelCache(device, ref_name);
+
+    if (entry.prog==0 && entry.ker==0) {
+
+        std::ostringstream options;
+        options << " -D T=" << dtype_traits<T>::getName()
+            << " -D To=" << dtype_traits<To>::getName()
+            << " -D THREADS=" << THREADS
+            << " -D FEAT_LEN=" << unroll_len;
+
+        switch(dist_type) {
+            case AF_SAD: options <<" -D DISTOP=_sad_"; break;
+            case AF_SSD: options <<" -D DISTOP=_ssd_"; break;
+            case AF_SHD: options <<" -D DISTOP=_shd_ -D __SHD__";
+                         break;
+            default: break;
+        }
+
+        if (std::is_same<T, double>::value ||
                 std::is_same<T, cdouble>::value) {
-                options << " -D USE_DOUBLE";
-            }
+            options << " -D USE_DOUBLE";
+        }
 
-            if (use_lmem)
-                options << " -D USE_LOCAL_MEM";
+        if (use_lmem)
+            options << " -D USE_LOCAL_MEM";
 
-            cl::Program prog;
-            buildProgram(prog,
-                          nearest_neighbour_cl,
-                          nearest_neighbour_cl_len,
-                          options.str());
+        cl::Program prog;
+        buildProgram(prog,
+                nearest_neighbour_cl,
+                nearest_neighbour_cl_len,
+                options.str());
 
-            entry.prog = new Program(prog);
-            entry.ker = new Kernel[3];
+        entry.prog = new Program(prog);
+        entry.ker = new Kernel[3];
 
-            entry.ker[0] = Kernel(*entry.prog, "nearest_neighbour_unroll");
-            entry.ker[1] = Kernel(*entry.prog, "nearest_neighbour");
-            entry.ker[2] = Kernel(*entry.prog, "select_matches");
+        entry.ker[0] = Kernel(*entry.prog, "nearest_neighbour_unroll");
+        entry.ker[1] = Kernel(*entry.prog, "nearest_neighbour");
+        entry.ker[2] = Kernel(*entry.prog, "select_matches");
 
-            kernelCaches[device][ref_name] = entry;
-    } else {
-        entry = cache_idx->second;
+        addKernelToCache(device, ref_name, entry);
     }
 
     const dim_t sample_dim = (dist_dim == 0) ? 1 : 0;
diff --git a/src/backend/opencl/kernel/random_engine.hpp b/src/backend/opencl/kernel/random_engine.hpp
index 8829b15a68..827f56e552 100644
--- a/src/backend/opencl/kernel/random_engine.hpp
+++ b/src/backend/opencl/kernel/random_engine.hpp
@@ -83,9 +83,10 @@ namespace opencl
                 "_" + string(dtype_traits<T>::getName()) +
                 "_" + to_string(kerIdx);
             int device = getActiveDeviceId();
-            kc_t::iterator idx = kernelCaches[device].find(ref_name);
-            kc_entry_t entry;
-            if (idx == kernelCaches[device].end()) {
+
+            kc_entry_t entry = kernelCache(device, ref_name);
+
+            if (entry.prog==0 && entry.ker==0) {
                 std::ostringstream options;
                 options << " -D T=" << dtype_traits<T>::getName()
                         << " -D THREADS=" << THREADS
@@ -104,9 +105,8 @@ namespace opencl
                 buildProgram(prog, 2, ker_strs, ker_lens, options.str());
                 entry.prog = new Program(prog);
                 entry.ker = new Kernel(*entry.prog, "generate");
-                kernelCaches[device][ref_name] = entry;
-            } else {
-                entry = idx->second;
+
+                addKernelToCache(device, ref_name, entry);
             }
 
             return *entry.ker;
@@ -121,17 +121,17 @@ namespace opencl
             int ker_len = random_engine_mersenne_init_cl_len;
             string ref_name = "mersenne_init";
             int device = getActiveDeviceId();
-            kc_t::iterator idx = kernelCaches[device].find(ref_name);
-            kc_entry_t entry;
-            if (idx == kernelCaches[device].end()) {
+
+            kc_entry_t entry = kernelCache(device, ref_name);
+
+            if (entry.prog==0 && entry.ker==0) {
                 std::string emptyOptionString;
                 cl::Program prog;
                 buildProgram(prog, 1, &ker_str, &ker_len, emptyOptionString);
                 entry.prog = new Program(prog);
                 entry.ker = new Kernel(*entry.prog, "initState");
-                kernelCaches[device][ref_name] = entry;
-            } else {
-                entry = idx->second;
+
+                addKernelToCache(device, ref_name, entry);
             }
 
             return *entry.ker;
diff --git a/src/backend/opencl/kernel/reduce.hpp b/src/backend/opencl/kernel/reduce.hpp
index 31038a3080..d9c67c90fe 100644
--- a/src/backend/opencl/kernel/reduce.hpp
+++ b/src/backend/opencl/kernel/reduce.hpp
@@ -61,10 +61,9 @@ namespace kernel
             std::to_string(threads_y);
 
         int device = getActiveDeviceId();
-        kc_t::iterator idx = kernelCaches[device].find(ref_name);
+        kc_entry_t entry = kernelCache(device, ref_name);
 
-        kc_entry_t entry;
-        if (idx == kernelCaches[device].end()) {
+        if (entry.prog==0 && entry.ker==0) {
             Binary<To, op> reduce;
             ToNumStr<To> toNumStr;
 
@@ -92,9 +91,7 @@ namespace kernel
             entry.prog = new Program(prog);
             entry.ker = new Kernel(*entry.prog, "reduce_dim_kernel");
 
-            kernelCaches[device][ref_name] = entry;
-        } else {
-            entry = idx->second;
+            addKernelToCache(device, ref_name, entry);
         }
 
         NDRange local(THREADS_X, threads_y);
@@ -179,10 +176,10 @@ namespace kernel
             std::to_string(threads_x);
 
         int device = getActiveDeviceId();
-        kc_t::iterator idx = kernelCaches[device].find(ref_name);
 
-        kc_entry_t entry;
-        if (idx == kernelCaches[device].end()) {
+        kc_entry_t entry = kernelCache(device, ref_name);
+
+        if (entry.prog==0 && entry.ker==0) {
 
             Binary<To, op> reduce;
             ToNumStr<To> toNumStr;
@@ -209,9 +206,7 @@ namespace kernel
             entry.prog = new Program(prog);
             entry.ker = new Kernel(*entry.prog, "reduce_first_kernel");
 
-            kernelCaches[device][ref_name] = entry;
-        } else {
-            entry = idx->second;
+            addKernelToCache(device, ref_name, entry);
         }
 
         NDRange local(threads_x, THREADS_PER_GROUP / threads_x);
diff --git a/src/backend/opencl/kernel/scan_dim.hpp b/src/backend/opencl/kernel/scan_dim.hpp
index 46ad2d1673..a48d271a5d 100644
--- a/src/backend/opencl/kernel/scan_dim.hpp
+++ b/src/backend/opencl/kernel/scan_dim.hpp
@@ -55,11 +55,10 @@ namespace kernel
             std::to_string(int(inclusive_scan));
 
         int device = getActiveDeviceId();
-        kc_t::iterator idx = kernelCaches[device].find(ref_name);
 
-        kc_entry_t entry;
-        if (idx == kernelCaches[device].end()) {
+        kc_entry_t entry = kernelCache(device, ref_name);
 
+        if (entry.prog==0 && entry.ker==0) {
             Binary<To, op> scan;
             ToNumStr<To> toNumStr;
 
@@ -91,10 +90,8 @@ namespace kernel
             entry.ker[0] = Kernel(*entry.prog, "scan_dim_kernel");
             entry.ker[1] = Kernel(*entry.prog, "bcast_dim_kernel");
 
-            kernelCaches[device][ref_name] = entry;
 
-        } else {
-            entry = idx->second;
+            addKernelToCache(device, ref_name, entry);
         }
 
         return entry.ker[kerIdx];
diff --git a/src/backend/opencl/kernel/scan_dim_by_key_impl.hpp b/src/backend/opencl/kernel/scan_dim_by_key_impl.hpp
index a8adbff7d3..217153d18c 100644
--- a/src/backend/opencl/kernel/scan_dim_by_key_impl.hpp
+++ b/src/backend/opencl/kernel/scan_dim_by_key_impl.hpp
@@ -57,10 +57,10 @@ namespace kernel
             std::to_string(int(inclusive_scan));
 
         int device = getActiveDeviceId();
-        kc_t::iterator idx = kernelCaches[device].find(ref_name);
 
-        kc_entry_t entry;
-        if (idx == kernelCaches[device].end()) {
+        kc_entry_t entry = kernelCache(device, ref_name);
+
+        if (entry.prog==0 && entry.ker==0) {
 
             Binary<To, op> scan;
             ToNumStr<To> toNumStr;
@@ -95,10 +95,7 @@ namespace kernel
             entry.ker[1] = Kernel(*entry.prog, "scan_dim_by_key_nonfinal_kernel");
             entry.ker[2] = Kernel(*entry.prog, "bcast_dim_kernel");
 
-            kernelCaches[device][ref_name] = entry;
-
-        } else {
-            entry = idx->second;
+            addKernelToCache(device, ref_name, entry);
         }
 
         return entry.ker[kerIdx];
diff --git a/src/backend/opencl/kernel/scan_first.hpp b/src/backend/opencl/kernel/scan_first.hpp
index 85f32ec2e9..58587c1ba9 100644
--- a/src/backend/opencl/kernel/scan_first.hpp
+++ b/src/backend/opencl/kernel/scan_first.hpp
@@ -56,11 +56,10 @@ namespace kernel
             std::to_string(int(inclusive_scan));
 
         int device = getActiveDeviceId();
-        kc_t::iterator idx = kernelCaches[device].find(ref_name);
 
-        kc_entry_t entry;
-        if (idx == kernelCaches[device].end()) {
+        kc_entry_t entry = kernelCache(device, ref_name);
 
+        if (entry.prog==0 && entry.ker==0) {
             const uint threads_y = THREADS_PER_GROUP / threads_x;
             const uint SHARED_MEM_SIZE = THREADS_PER_GROUP;
 
@@ -95,10 +94,7 @@ namespace kernel
             entry.ker[0] = Kernel(*entry.prog, "scan_first_kernel");
             entry.ker[1] = Kernel(*entry.prog, "bcast_first_kernel");
 
-            kernelCaches[device][ref_name] = entry;
-
-        } else {
-            entry = idx->second;
+            addKernelToCache(device, ref_name, entry);
         }
 
         return entry.ker[kerIdx];
diff --git a/src/backend/opencl/kernel/scan_first_by_key_impl.hpp b/src/backend/opencl/kernel/scan_first_by_key_impl.hpp
index 9417c1a777..771dfca8f4 100644
--- a/src/backend/opencl/kernel/scan_first_by_key_impl.hpp
+++ b/src/backend/opencl/kernel/scan_first_by_key_impl.hpp
@@ -58,10 +58,9 @@ namespace kernel
             std::to_string(int(inclusive_scan));
 
         int device = getActiveDeviceId();
-        kc_t::iterator idx = kernelCaches[device].find(ref_name);
+        kc_entry_t entry = kernelCache(device, ref_name);
 
-        kc_entry_t entry;
-        if (idx == kernelCaches[device].end()) {
+        if (entry.prog==0 && entry.ker==0) {
 
             const uint threads_y = THREADS_PER_GROUP / threads_x;
             const uint SHARED_MEM_SIZE = THREADS_PER_GROUP;
@@ -99,10 +98,7 @@ namespace kernel
             entry.ker[1] = Kernel(*entry.prog, "scan_first_by_key_nonfinal_kernel");
             entry.ker[2] = Kernel(*entry.prog, "bcast_first_kernel");
 
-            kernelCaches[device][ref_name] = entry;
-
-        } else {
-            entry = idx->second;
+            addKernelToCache(device, ref_name, entry);
         }
 
         return entry.ker[kerIdx];
diff --git a/src/backend/opencl/kernel/sparse.hpp b/src/backend/opencl/kernel/sparse.hpp
index df56d5fedb..67c2650bcb 100644
--- a/src/backend/opencl/kernel/sparse.hpp
+++ b/src/backend/opencl/kernel/sparse.hpp
@@ -50,10 +50,9 @@ namespace opencl
                 std::to_string(REPEAT);
 
             int device = getActiveDeviceId();
-            auto idx = kernelCaches[device].find(ref_name);
-            kc_entry_t entry;
+            kc_entry_t entry = kernelCache(device, ref_name);
 
-            if (idx == kernelCaches[device].end()) {
+            if (entry.prog==0 && entry.ker==0) {
                 std::ostringstream options;
                 options << " -D T="        << dtype_traits<T>::getName()
                         << " -D reps="     << REPEAT
@@ -68,8 +67,8 @@ namespace opencl
                 buildProgram(prog, coo2dense_cl, coo2dense_cl_len, options.str());
                 entry.prog   = new Program(prog);
                 entry.ker = new Kernel(*entry.prog, "coo2dense_kernel");
-            } else {
-                entry = idx->second;
+
+                addKernelToCache(device, ref_name, entry);
             };
 
             auto coo2denseOp = KernelFunctor<Buffer, const KParam,
@@ -106,10 +105,9 @@ namespace opencl
                 std::to_string(threads);
 
             int device = getActiveDeviceId();
-            auto idx = kernelCaches[device].find(ref_name);
-            kc_entry_t entry;
+            kc_entry_t entry = kernelCache(device, ref_name);
 
-            if (idx == kernelCaches[device].end()) {
+            if (entry.prog==0 && entry.ker==0) {
 
                 std::ostringstream options;
                 options << " -D T=" << dtype_traits<T>::getName();
@@ -127,8 +125,8 @@ namespace opencl
                 buildProgram(prog, 1, ker_strs, ker_lens, options.str());
                 entry.prog = new Program(prog);
                 entry.ker  = new Kernel(*entry.prog, "csr2dense");
-            } else {
-                entry = idx->second;
+
+                addKernelToCache(device, ref_name, entry);
             }
 
             NDRange local(threads, 1);
@@ -196,10 +194,9 @@ namespace opencl
                 std::string(dtype_traits<T>::getName());
 
             int device = getActiveDeviceId();
-            auto idx = kernelCaches[device].find(ref_name);
-            kc_entry_t entry;
+            kc_entry_t entry = kernelCache(device, ref_name);
 
-            if (idx == kernelCaches[device].end()) {
+            if (entry.prog==0 && entry.ker==0) {
 
                 std::ostringstream options;
                 options << " -D T=" << dtype_traits<T>::getName();
@@ -222,9 +219,7 @@ namespace opencl
                 entry.prog = new Program(prog);
                 entry.ker  = new Kernel(*entry.prog, "dense2csr_split_kernel");
 
-                kernelCaches[device][ref_name] = entry;
-            } else {
-                entry = idx->second;
+                addKernelToCache(device, ref_name, entry);
             }
 
             NDRange local(THREADS_X, THREADS_Y);
@@ -258,10 +253,9 @@ namespace opencl
                 std::string(dtype_traits<T>::getName());
 
             int device = getActiveDeviceId();
-            auto idx = kernelCaches[device].find(ref_name);
-            kc_entry_t entry;
+            kc_entry_t entry = kernelCache(device, ref_name);
 
-            if (idx == kernelCaches[device].end()) {
+            if (entry.prog==0 && entry.ker==0) {
                 std::ostringstream options;
                 options << " -D T="        << dtype_traits<T>::getName();
 
@@ -274,8 +268,8 @@ namespace opencl
                 buildProgram(prog, csr2coo_cl, csr2coo_cl_len, options.str());
                 entry.prog   = new Program(prog);
                 entry.ker = new Kernel(*entry.prog, "swapIndex_kernel");
-            } else {
-                entry = idx->second;
+
+                addKernelToCache(device, ref_name, entry);
             };
 
             auto swapIndexOp = KernelFunctor<Buffer, Buffer,
@@ -307,10 +301,9 @@ namespace opencl
                 std::string(dtype_traits<T>::getName());
 
             int device = getActiveDeviceId();
-            auto idx = kernelCaches[device].find(ref_name);
-            kc_entry_t entry;
+            kc_entry_t entry = kernelCache(device, ref_name);
 
-            if (idx == kernelCaches[device].end()) {
+            if (entry.prog==0 && entry.ker==0) {
 
                 std::ostringstream options;
                 options << " -D T=" << dtype_traits<T>::getName();
@@ -327,8 +320,8 @@ namespace opencl
                 buildProgram(prog, 1, ker_strs, ker_lens, options.str());
                 entry.prog = new Program(prog);
                 entry.ker  = new Kernel(*entry.prog, "csr2coo");
-            } else {
-                entry = idx->second;
+
+                addKernelToCache(device, ref_name, entry);
             }
 
             cl::Buffer *scratch = bufferAlloc(orowIdx.info.dims[0] * sizeof(int));
@@ -374,10 +367,9 @@ namespace opencl
                 std::string(dtype_traits<T>::getName());
 
             int device = getActiveDeviceId();
-            auto idx = kernelCaches[device].find(ref_name);
-            kc_entry_t entry;
+            kc_entry_t entry = kernelCache(device, ref_name);
 
-            if (idx == kernelCaches[device].end()) {
+            if (entry.prog==0 && entry.ker==0) {
                 std::ostringstream options;
                 options << " -D T="        << dtype_traits<T>::getName();
 
@@ -390,8 +382,8 @@ namespace opencl
                 buildProgram(prog, csr2coo_cl, csr2coo_cl_len, options.str());
                 entry.prog   = new Program(prog);
                 entry.ker = new Kernel(*entry.prog, "csrReduce_kernel");
-            } else {
-                entry = idx->second;
+
+                addKernelToCache(device, ref_name, entry);
             };
 
             auto csrReduceOp = KernelFunctor<Buffer, const Buffer, const int, const int> (*entry.ker);
diff --git a/src/backend/opencl/kernel/transform.hpp b/src/backend/opencl/kernel/transform.hpp
index dc38696d8d..b27445e603 100644
--- a/src/backend/opencl/kernel/transform.hpp
+++ b/src/backend/opencl/kernel/transform.hpp
@@ -71,10 +71,9 @@ namespace opencl
                 std::to_string(order);
 
             int device = getActiveDeviceId();
-            auto idx = kernelCaches[device].find(ref_name);
-            kc_entry_t entry;
+            kc_entry_t entry = kernelCache(device, ref_name);
 
-            if (idx == kernelCaches[device].end()) {
+            if (entry.prog==0 && entry.ker==0) {
                 ToNumStr<T> toNumStr;
                 std::ostringstream options;
                 options << " -D T="           << dtype_traits<T>::getName()
@@ -106,8 +105,8 @@ namespace opencl
                 buildProgram(prog, 2, ker_strs, ker_lens, options.str());
                 entry.prog = new Program(prog);
                 entry.ker = new Kernel(*entry.prog, "transform_kernel");
-            } else {
-                entry = idx->second;
+
+                addKernelToCache(device, ref_name, entry);
             }
 
             auto transformOp = KernelFunctor<Buffer, const KParam,
diff --git a/src/backend/opencl/kernel/unwrap.hpp b/src/backend/opencl/kernel/unwrap.hpp
index 0ae3b7002a..100bd9ca66 100644
--- a/src/backend/opencl/kernel/unwrap.hpp
+++ b/src/backend/opencl/kernel/unwrap.hpp
@@ -48,11 +48,10 @@ namespace opencl
                 std::to_string(is_column);
 
             int device = getActiveDeviceId();
-            kc_t::iterator idx = kernelCaches[device].find(ref_name);
 
-            kc_entry_t entry;
-            if (idx == kernelCaches[device].end()) {
+            kc_entry_t entry = kernelCache(device, ref_name);
 
+            if (entry.prog==0 && entry.ker==0) {
                 ToNumStr<T> toNumStr;
                 std::ostringstream options;
                 options << " -D is_column=" << is_column
@@ -70,9 +69,7 @@ namespace opencl
                 entry.prog = new Program(prog);
                 entry.ker = new Kernel(*entry.prog, "unwrap_kernel");
 
-                kernelCaches[device][ref_name] = entry;
-            } else {
-                entry = idx->second;
+                addKernelToCache(device, ref_name, entry);
             }
 
             dim_t TX = 1, TY = 1;
diff --git a/src/backend/opencl/kernel/wrap.hpp b/src/backend/opencl/kernel/wrap.hpp
index 3e35a2fbea..5ad2efe38c 100644
--- a/src/backend/opencl/kernel/wrap.hpp
+++ b/src/backend/opencl/kernel/wrap.hpp
@@ -48,10 +48,9 @@ namespace opencl
                 std::to_string(is_column);
 
             int device = getActiveDeviceId();
-            kc_t::iterator idx = kernelCaches[device].find(ref_name);
+            kc_entry_t entry = kernelCache(device, ref_name);
 
-            kc_entry_t entry;
-            if (idx == kernelCaches[device].end()) {
+            if (entry.prog==0 && entry.ker==0) {
 
                 ToNumStr<T> toNumStr;
                 std::ostringstream options;
@@ -70,9 +69,7 @@ namespace opencl
                 entry.prog = new Program(prog);
                 entry.ker = new Kernel(*entry.prog, "wrap_kernel");
 
-                kernelCaches[device][ref_name] = entry;
-            } else {
-                entry = idx->second;
+                addKernelToCache(device, ref_name, entry);
             }
 
             dim_t nx = (out.info.dims[0] + 2 * px - wx) / sx + 1;
diff --git a/src/backend/opencl/platform.cpp b/src/backend/opencl/platform.cpp
index 67dd594e08..7acd2baeeb 100644
--- a/src/backend/opencl/platform.cpp
+++ b/src/backend/opencl/platform.cpp
@@ -34,6 +34,7 @@
 #include <version.hpp>
 
 #include <algorithm>
+#include <boost/thread.hpp>
 #include <cctype>
 #include <functional>
 #include <map>
@@ -54,8 +55,16 @@ using cl::Context;
 using cl::CommandQueue;
 using cl::Device;
 
+typedef boost::shared_mutex smutex_t;
+typedef boost::shared_lock<smutex_t> rlock_t;
+typedef boost::unique_lock<smutex_t> wlock_t;
+typedef boost::upgrade_lock<smutex_t> ulock_t;
+typedef boost::upgrade_to_unique_lock<smutex_t> u2ulock_t;
+
 namespace opencl
 {
+static smutex_t kernelCacheMutexes[DeviceManager::MAX_DEVICES];
+
 #if defined (OS_MAC)
 static const std::string CL_GL_SHARING_EXT = "cl_APPLE_gl_sharing";
 #else
@@ -629,6 +638,33 @@ PlanCache& fftManager()
     return clfftManagers[getActiveDeviceId()];
 }
 
+void addKernelToCache(int device, const std::string& key, const kc_entry_t entry)
+{
+    wlock_t lock(kernelCacheMutexes[device]);
+
+    DeviceManager::getInstance().kernelCaches[device].emplace(key, entry);
+}
+
+void removeKernelFromCache(int device, const std::string& key)
+{
+    wlock_t lock(kernelCacheMutexes[device]);
+
+    DeviceManager::getInstance().kernelCaches[device].erase(key);
+}
+
+kc_entry_t kernelCache(int device, const std::string& key)
+{
+    DeviceManager& inst = DeviceManager::getInstance();
+
+    rlock_t lock(kernelCacheMutexes[device]);
+
+    kc_t::iterator iter = inst.kernelCaches[device].find(key);
+    if (iter == inst.kernelCaches[device].end()) {
+        return kc_entry_t{0, 0};
+    } else
+        return iter->second;
+}
+
 DeviceManager& DeviceManager::getInstance()
 {
     static DeviceManager my_instance;
diff --git a/src/backend/opencl/platform.hpp b/src/backend/opencl/platform.hpp
index 66a1acb53a..de55c38947 100644
--- a/src/backend/opencl/platform.hpp
+++ b/src/backend/opencl/platform.hpp
@@ -26,6 +26,7 @@
 #include <vector>
 #include <string>
 
+#include <cache.hpp>
 #include <memory.hpp>
 #include <GraphicsResourceManager.hpp>
 #include <clfft.hpp>
@@ -93,6 +94,12 @@ MemoryManagerPinned& pinnedMemoryManager();
 GraphicsResourceManager& interopManager();
 
 PlanCache& fftManager();
+
+void addKernelToCache(int device, const std::string& key, const kc_entry_t entry);
+
+void removeKernelFromCache(int device, const std::string& key);
+
+kc_entry_t kernelCache(int device, const std::string& key);
 //
 ///////////////////////// END Sub-Managers /////////////////////
 
@@ -106,6 +113,12 @@ class DeviceManager
 
     friend PlanCache& fftManager();
 
+    friend void addKernelToCache(int device, const std::string& key, const kc_entry_t entry);
+
+    friend void removeKernelFromCache(int device, const std::string& key);
+
+    friend kc_entry_t kernelCache(int device, const std::string& key);
+
     friend std::string getDeviceInfo();
 
     friend int getDeviceCount();
@@ -181,5 +194,7 @@ class DeviceManager
 
         std::unique_ptr<GraphicsResourceManager> gfxManagers[MAX_DEVICES];
         clfftSetupData mFFTSetup;
+
+        kc_t kernelCaches[MAX_DEVICES];
 };
 }

From c9f53fc264a1a2f824798c94a6104443820562cd Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Mon, 20 Feb 2017 22:53:34 +0530
Subject: [PATCH 38/82] Add thread safety to CUDA JIT kernel cache

---
 src/backend/cuda/jit.cpp | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/src/backend/cuda/jit.cpp b/src/backend/cuda/jit.cpp
index 7d4111874a..1ec2e12b62 100644
--- a/src/backend/cuda/jit.cpp
+++ b/src/backend/cuda/jit.cpp
@@ -42,12 +42,17 @@
 #include <math.hpp>
 #include <nvvm.h>
 
+#include <boost/thread.hpp>
 #include <functional>
 #include <map>
 #include <memory>
 #include <stdexcept>
 #include <vector>
 
+typedef boost::shared_mutex smutex_t;
+typedef boost::upgrade_lock<smutex_t> ulock_t;
+typedef boost::upgrade_to_unique_lock<smutex_t> u2ulock_t;
+
 namespace cuda
 {
 
@@ -530,17 +535,21 @@ static kc_entry_t compileKernel(const char *ker_name, string jit_ker)
 
 static CUfunction getKernel(vector<Node *> nodes, bool is_linear)
 {
+    typedef std::map<std::string, kc_entry_t> kc_t;
+
+    static smutex_t mutexes[DeviceManager::MAX_DEVICES];
+    static kc_t kernelCaches[DeviceManager::MAX_DEVICES];
 
     string funcName = getFuncName(nodes, is_linear);
+    int device      = getActiveDeviceId();
 
-    typedef map<string, kc_entry_t> kc_t;
-    static kc_t kernelCaches[DeviceManager::MAX_DEVICES];
-    int device = getActiveDeviceId();
+    ulock_t lock(mutexes[device]);
 
     kc_t::iterator idx = kernelCaches[device].find(funcName);
     kc_entry_t entry = {NULL, NULL};
 
     if (idx == kernelCaches[device].end()) {
+        u2ulock_t unqLock(lock);
         string jit_ker = getKernelString(funcName, nodes, is_linear);
         entry = compileKernel(funcName.c_str(), jit_ker);
         kernelCaches[device][funcName] = entry;

From 49793ed87d017a298707c99e152598db569a11b6 Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Tue, 21 Feb 2017 16:51:54 +0530
Subject: [PATCH 39/82] Make ForgeManager(gfx upstream) interface thread safe

---
 src/api/c/graphics_common.cpp | 172 +++++++++++++++++++++++-----------
 src/api/c/graphics_common.hpp |   9 +-
 2 files changed, 121 insertions(+), 60 deletions(-)

diff --git a/src/api/c/graphics_common.cpp b/src/api/c/graphics_common.cpp
index b35d15b879..ec16030541 100644
--- a/src/api/c/graphics_common.cpp
+++ b/src/api/c/graphics_common.cpp
@@ -15,10 +15,18 @@
 #include <backend.hpp>
 #include <platform.hpp>
 #include <util.hpp>
+#include <mutex>
+#include <boost/thread.hpp>
 
 using namespace std;
 using namespace gl;
 
+typedef boost::shared_mutex smutex_t;
+typedef boost::shared_lock<smutex_t> rlock_t;
+typedef boost::unique_lock<smutex_t> wlock_t;
+typedef boost::upgrade_lock<smutex_t> ulock_t;
+typedef boost::upgrade_to_unique_lock<smutex_t> u2ulock_t;
+
 template<typename T>
 gl::GLenum getGLType() { return GL_FLOAT; }
 
@@ -171,6 +179,12 @@ double step_round(const double in, const bool dir)
 
 namespace graphics
 {
+static smutex_t  gImgMapMutex;
+static smutex_t  gPltMapMutex;
+static smutex_t  gHstMapMutex;
+static smutex_t  gSfcMapMutex;
+static smutex_t  gVcfMapMutex;
+static smutex_t  gChartMutex;
 
 ForgeManager& ForgeManager::getInstance()
 {
@@ -180,55 +194,75 @@ ForgeManager& ForgeManager::getInstance()
 
 ForgeManager::~ForgeManager()
 {
-    destroyResources();
+    /* clear all OpenGL resource objects (images, plots, histograms etc) first
+     * and then delete the windows */
+    for(ImgMapIter iter = mImgMap.begin(); iter != mImgMap.end(); iter++)
+        delete (iter->second);
+
+    for(PltMapIter iter = mPltMap.begin(); iter != mPltMap.end(); iter++)
+        delete (iter->second);
+
+    for(HstMapIter iter = mHstMap.begin(); iter != mHstMap.end(); iter++)
+        delete (iter->second);
+
+    for(ChartMapIter iter = mChartMap.begin(); iter != mChartMap.end(); iter++) {
+        for(int i = 0; i < (int)(iter->second).size(); i++) {
+            if((iter->second)[i] != NULL) {
+                delete (iter->second)[i];
+                mChartAxesOverrideMap.erase((iter->second)[i]);
+            }
+        }
+    }
 }
 
-forge::Font* ForgeManager::getFont(const bool dontCreate)
+forge::Font* ForgeManager::getFont()
 {
-    static bool flag = true;
-    static forge::Font* fnt = NULL;
+    static std::once_flag flag;
+    static std::unique_ptr<forge::Font> fnt;
 
     CheckGL("Begin ForgeManager::getFont");
-
-    if (flag && !dontCreate) {
-        fnt = new forge::Font();
+    std::call_once(flag,
+            [] {
+                fnt.reset(new forge::Font());
 #if defined(_WIN32) || defined(_MSC_VER)
-        fnt->loadSystemFont("Arial");
+                fnt->loadSystemFont("Arial");
 #else
-        fnt->loadSystemFont("Vera");
+                fnt->loadSystemFont("Vera");
 #endif
-        CheckGL("End ForgeManager::getFont");
-        flag = false;
-    };
+            });
+    CheckGL("End ForgeManager::getFont");
 
-    return fnt;
+    return fnt.get();
 }
 
-forge::Window* ForgeManager::getMainWindow(const bool dontCreate)
+forge::Window* ForgeManager::getMainWindow()
 {
-    static bool flag = true;
-    static forge::Window* wnd = NULL;
+    static std::once_flag flag;
+    static std::unique_ptr<forge::Window> wnd;
 
+    CheckGL("Begin ForgeManager::getMainWindow");
     // Define AF_DISABLE_GRAPHICS with any value to disable initialization
     std::string noGraphicsENV = getEnvVar("AF_DISABLE_GRAPHICS");
-    if(noGraphicsENV.empty()) { // If AF_DISABLE_GRAPHICS is not defined
-        if (flag && !dontCreate) {
-            wnd = new forge::Window(WIDTH, HEIGHT, "ArrayFire", NULL, true);
-            makeContextCurrent(wnd);
 
-            ForgeManager& fgMngr = ForgeManager::getInstance();
-            fgMngr.setWindowChartGrid(wnd, 1, 1);
+    if (noGraphicsENV.empty()) { // If AF_DISABLE_GRAPHICS is not defined
+        std::call_once(flag,
+                [] {
+                    wnd.reset(new forge::Window(WIDTH, HEIGHT, "ArrayFire", NULL, true));
+                    makeContextCurrent(wnd.get());
 
-            CheckGL("End ForgeManager::getMainWindow");
-            flag = false;
-        };
+                    ForgeManager::getInstance().setWindowChartGrid(wnd.get(), 1, 1);
+                });
     }
-    return wnd;
+    CheckGL("End ForgeManager::getMainWindow");
+
+    return wnd.get();
 }
 
 void ForgeManager::setWindowChartGrid(const forge::Window* window,
                                       const int r, const int c)
 {
+    wlock_t lock(gChartMutex);
+
     ChartMapIter iter = mChartMap.find(window);
 
     if(iter != mChartMap.end()) {
@@ -254,16 +288,22 @@ void ForgeManager::setWindowChartGrid(const forge::Window* window,
 forge::Chart* ForgeManager::getChart(const forge::Window* window, const int r, const int c,
                                      const forge::ChartType ctype)
 {
+    ulock_t lock(gChartMutex);
+
     forge::Chart* chart = NULL;
     ChartMapIter iter = mChartMap.find(window);
 
-    if(iter != mChartMap.end()) {
+    if (iter != mChartMap.end()) {
+
         int gRows = window->gridRows();
         int gCols = window->gridCols();
 
         if(c >= gCols || r >= gRows)
             AF_ERROR("Grid points are out of bounds", AF_ERR_TYPE);
 
+        // upgrade to exclusive access to make changes
+        u2ulock_t unqLock(lock);
+
         chart = (iter->second)[c * gRows + r];
 
         if (chart == NULL) {
@@ -301,9 +341,16 @@ forge::Image* ForgeManager::getImage(int w, int h, forge::ChannelFormat mode, fo
     key = (((key << 16) | mode) << 16) | type;
 
     ChartKey_t keypair = std::make_pair(key, nullptr);
+
+    ulock_t lock(gImgMapMutex);
+
     ImgMapIter iter = mImgMap.find(keypair);
+
     if (iter==mImgMap.end()) {
         forge::Image* temp = new forge::Image(w, h, mode, type);
+
+        u2ulock_t unqLock(lock);
+
         mImgMap[keypair] = temp;
     }
 
@@ -324,13 +371,21 @@ forge::Image* ForgeManager::getImage(forge::Chart* chart, int w, int h,
     key = (((key << 16) | mode) << 16) | type;
 
     ChartKey_t keypair = std::make_pair(key, chart);
+
+    ulock_t lock(gImgMapMutex);
+
     ImgMapIter iter = mImgMap.find(keypair);
+
     if (iter==mImgMap.end()) {
         if(chart->getChartType() != FG_CHART_2D)
             AF_ERROR("Image can only be added to chart of type FG_CHART_2D", AF_ERR_TYPE);
 
         forge::Image* temp = new forge::Image(w, h, mode, type);
+
+        u2ulock_t unqLock(lock);
+
         mImgMap[keypair] = temp;
+
         chart->add(*mImgMap[keypair]);
     }
 
@@ -350,10 +405,18 @@ forge::Plot* ForgeManager::getPlot(forge::Chart* chart, int nPoints, forge::dtyp
     key |= (((((dtype & 0x000F) << 12) | (ptype & 0x000F)) << 8) | (mtype & 0x000F));
 
     ChartKey_t keypair = std::make_pair(key, chart);
+
+    ulock_t lock(gPltMapMutex);
+
     PltMapIter iter = mPltMap.find(keypair);
+
     if (iter==mPltMap.end()) {
         forge::Plot* temp = new forge::Plot(nPoints, dtype, chart->getChartType(), ptype, mtype);
+
+        u2ulock_t unqLock(lock);
+
         mPltMap[keypair] = temp;
+
         chart->add(*mPltMap[keypair]);
     }
 
@@ -371,13 +434,21 @@ forge::Histogram* ForgeManager::getHistogram(forge::Chart* chart, int nBins, for
     long long key = ((nBins & _48BIT) << 48) | (type & _16BIT);
 
     ChartKey_t keypair = std::make_pair(key, chart);
+
+    ulock_t lock(gHstMapMutex);
+
     HstMapIter iter = mHstMap.find(keypair);
+
     if (iter==mHstMap.end()) {
         if(chart->getChartType() != FG_CHART_2D)
             AF_ERROR("Histogram can only be added to chart of type FG_CHART_2D", AF_ERR_TYPE);
 
         forge::Histogram* temp = new forge::Histogram(nBins, type);
+
+        u2ulock_t unqLock(lock);
+
         mHstMap[keypair] = temp;
+
         chart->add(*mHstMap[keypair]);
     }
 
@@ -395,13 +466,21 @@ forge::Surface* ForgeManager::getSurface(forge::Chart* chart, int nX, int nY, fo
     long long key = (((nX * nY) & _48BIT) << 48) | (type & _16BIT);
 
     ChartKey_t keypair = std::make_pair(key, chart);
+
+    ulock_t lock(gSfcMapMutex);
+
     SfcMapIter iter = mSfcMap.find(keypair);
+
     if (iter==mSfcMap.end()) {
         if(chart->getChartType() != FG_CHART_3D)
             AF_ERROR("Surface can only be added to chart of type FG_CHART_3D", AF_ERR_TYPE);
 
         forge::Surface* temp = new forge::Surface(nX, nY, type);
+
+        u2ulock_t unqLock(lock);
+
         mSfcMap[keypair] = temp;
+
         chart->add(*mSfcMap[keypair]);
     }
 
@@ -419,10 +498,18 @@ forge::VectorField* ForgeManager::getVectorField(forge::Chart* chart, int nPoint
     long long key = (((nPoints) & _48BIT) << 48) | (type & _16BIT);
 
     ChartKey_t keypair = std::make_pair(key, chart);
+
+    ulock_t lock(gVcfMapMutex);
+
     VcfMapIter iter = mVcfMap.find(keypair);
+
     if (iter==mVcfMap.end()) {
         forge::VectorField* temp = new forge::VectorField(nPoints, type, chart->getChartType());
+
+        u2ulock_t unqLock(lock);
+
         mVcfMap[keypair] = temp;
+
         chart->add(*mVcfMap[keypair]);
     }
 
@@ -431,6 +518,8 @@ forge::VectorField* ForgeManager::getVectorField(forge::Chart* chart, int nPoint
 
 bool ForgeManager::getChartAxesOverride(forge::Chart* chart)
 {
+    rlock_t lock(gChartMutex);
+
     ChartAxesOverrideIter iter = mChartAxesOverrideMap.find(chart);
     if (iter == mChartAxesOverrideMap.end()) {
         AF_ERROR("Chart Not Found!", AF_ERR_INTERNAL);
@@ -440,39 +529,14 @@ bool ForgeManager::getChartAxesOverride(forge::Chart* chart)
 
 void ForgeManager::setChartAxesOverride(forge::Chart* chart, bool flag)
 {
+    rlock_t lock(gChartMutex);
+
     ChartAxesOverrideIter iter = mChartAxesOverrideMap.find(chart);
     if (iter == mChartAxesOverrideMap.end()) {
         AF_ERROR("Chart Not Found!", AF_ERR_INTERNAL);
     }
     mChartAxesOverrideMap[chart] = flag;
 }
-
-void ForgeManager::destroyResources()
-{
-    /* clear all OpenGL resource objects (images, plots, histograms etc) first
-     * and then delete the windows */
-    for(ImgMapIter iter = mImgMap.begin(); iter != mImgMap.end(); iter++)
-        delete (iter->second);
-
-    for(PltMapIter iter = mPltMap.begin(); iter != mPltMap.end(); iter++)
-        delete (iter->second);
-
-    for(HstMapIter iter = mHstMap.begin(); iter != mHstMap.end(); iter++)
-        delete (iter->second);
-
-    for(ChartMapIter iter = mChartMap.begin(); iter != mChartMap.end(); iter++) {
-        for(int i = 0; i < (int)(iter->second).size(); i++) {
-            if((iter->second)[i] != NULL) {
-                delete (iter->second)[i];
-                mChartAxesOverrideMap.erase((iter->second)[i]);
-            }
-        }
-    }
-
-    delete getFont(true);
-    delete getMainWindow(true);
-}
-
 }
 
 #endif
diff --git a/src/api/c/graphics_common.hpp b/src/api/c/graphics_common.hpp
index ac5ebe7709..b8f29a343a 100644
--- a/src/api/c/graphics_common.hpp
+++ b/src/api/c/graphics_common.hpp
@@ -43,7 +43,6 @@ double step_round(const double in, const bool dir);
 
 namespace graphics
 {
-
 enum Defaults {
     WIDTH = 1280,
     HEIGHT= 720
@@ -104,8 +103,8 @@ class ForgeManager
         static ForgeManager& getInstance();
         ~ForgeManager();
 
-        forge::Font*    getFont(const bool dontCreate=false);
-        forge::Window*  getMainWindow(const bool dontCreate=false);
+        forge::Font*    getFont();
+        forge::Window*  getMainWindow();
 
         void            setWindowChartGrid(const forge::Window* window,
                                            const int r, const int c);
@@ -130,11 +129,9 @@ class ForgeManager
         ForgeManager() {}
         ForgeManager(ForgeManager const&);
         void operator=(ForgeManager const&);
-        void destroyResources();
 };
-
 }
 
-#define MAIN_WINDOW graphics::ForgeManager::getInstance().getMainWindow(true)
+#define MAIN_WINDOW graphics::ForgeManager::getInstance().getMainWindow()
 
 #endif

From d45c4c86de5942c15e6f15a3e4edaa9767476958 Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Tue, 21 Feb 2017 21:00:39 +0530
Subject: [PATCH 40/82] Omit thread_local qualifier for fft cache objects on
 OSX

XCode's Clang doesn't support thread_local qualifier yet.
---
 src/backend/cuda/platform.cpp   | 8 ++++++++
 src/backend/opencl/platform.cpp | 8 ++++++++
 2 files changed, 16 insertions(+)

diff --git a/src/backend/cuda/platform.cpp b/src/backend/cuda/platform.cpp
index 847be47c3c..68d0cb502b 100644
--- a/src/backend/cuda/platform.cpp
+++ b/src/backend/cuda/platform.cpp
@@ -421,7 +421,15 @@ GraphicsResourceManager& interopManager()
 
 PlanCache& fftManager()
 {
+    //FIXME Change to better check later, may be Clang version based check
+#if defined(OS_MAC)
+    // XCode Clang doesn't support thread_local qualifier
+    // Hence, making the cache manager per device
+    static PlanCache cufftManagers[DeviceManager::MAX_DEVICES];
+#else
+    //Otherwise, cache manager is per thread per devicea, less congestion
     thread_local static PlanCache cufftManagers[DeviceManager::MAX_DEVICES];
+#endif
 
     return cufftManagers[getActiveDeviceId()];
 }
diff --git a/src/backend/opencl/platform.cpp b/src/backend/opencl/platform.cpp
index 7acd2baeeb..4b43a6865a 100644
--- a/src/backend/opencl/platform.cpp
+++ b/src/backend/opencl/platform.cpp
@@ -633,7 +633,15 @@ GraphicsResourceManager& interopManager()
 
 PlanCache& fftManager()
 {
+    //FIXME Change to better check later, may be Clang version based check
+#if defined(OS_MAC)
+    // XCode Clang doesn't support thread_local qualifier
+    // Hence, making the cache manager per device
+    static PlanCache clfftManagers[DeviceManager::MAX_DEVICES];
+#else
+    //Otherwise, cache manager is per thread per devicea, less congestion
     thread_local static PlanCache clfftManagers[DeviceManager::MAX_DEVICES];
+#endif
 
     return clfftManagers[getActiveDeviceId()];
 }

From 60a325906385f80c7f256f58a576a69d204ae414 Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Wed, 22 Feb 2017 18:16:57 +0530
Subject: [PATCH 41/82] Fix memory management issue for user added opencl
 devices

Earlier, for user added devices amid the program execution, the
memory management wasn't being properly initialised, thereby causing
segmentation faults in such cases. This has been taken care of now.
---
 src/api/c/graphics_common.cpp        |   2 -
 src/backend/common/MemoryManager.hpp | 108 +++++++++++++++++----------
 src/backend/opencl/platform.cpp      |   5 ++
 test/ocl_ext_context.cpp             |  40 +++++-----
 4 files changed, 96 insertions(+), 59 deletions(-)

diff --git a/src/api/c/graphics_common.cpp b/src/api/c/graphics_common.cpp
index ec16030541..88a8f8e230 100644
--- a/src/api/c/graphics_common.cpp
+++ b/src/api/c/graphics_common.cpp
@@ -240,7 +240,6 @@ forge::Window* ForgeManager::getMainWindow()
     static std::once_flag flag;
     static std::unique_ptr<forge::Window> wnd;
 
-    CheckGL("Begin ForgeManager::getMainWindow");
     // Define AF_DISABLE_GRAPHICS with any value to disable initialization
     std::string noGraphicsENV = getEnvVar("AF_DISABLE_GRAPHICS");
 
@@ -253,7 +252,6 @@ forge::Window* ForgeManager::getMainWindow()
                     ForgeManager::getInstance().setWindowChartGrid(wnd.get(), 1, 1);
                 });
     }
-    CheckGL("End ForgeManager::getMainWindow");
 
     return wnd.get();
 }
diff --git a/src/backend/common/MemoryManager.hpp b/src/backend/common/MemoryManager.hpp
index 892df3f50c..219385ffd2 100644
--- a/src/backend/common/MemoryManager.hpp
+++ b/src/backend/common/MemoryManager.hpp
@@ -42,7 +42,7 @@ class MemoryManager
     typedef std::unordered_map<size_t, std::vector<void *> >free_t;
     typedef free_t::iterator free_iter;
 
-    typedef struct
+    typedef struct memory_info
     {
         locked_t locked_map;
         free_t   free_map;
@@ -52,6 +52,17 @@ class MemoryManager
         size_t total_bytes;
         size_t total_buffers;
         size_t max_bytes;
+
+        memory_info()
+        {
+            // Calling getMaxMemorySize() here calls the virtual function that returns 0
+            // Call it from outside the constructor.
+            max_bytes     = ONE_GB;
+            total_bytes   = 0;
+            total_buffers = 0;
+            lock_bytes    = 0;
+            lock_buffers  = 0;
+        }
     } memory_info;
 
     size_t mem_step_size;
@@ -74,38 +85,75 @@ class MemoryManager
         return static_cast<T*>(this)->getMaxMemorySize(id);
     }
 
+    void cleanDeviceMemoryManager(int device)
+    {
+        if (this->debug_mode) return;
+
+        lock_guard_t lock(this->memory_mutex);
+        memory_info& current = memory[device];
+
+        // Return if all buffers are locked
+        if (current.total_buffers == current.lock_buffers) return;
+
+        for (auto &kv : current.free_map) {
+            size_t num_ptrs = kv.second.size();
+            //Free memory by popping the last element
+            for (int n = num_ptrs-1; n >= 0; n--) {
+                this->nativeFree(kv.second[n]);
+                current.total_bytes -= kv.first;
+                current.total_buffers--;
+                kv.second.pop_back();
+            }
+        }
+        current.free_map.clear();
+    }
+
     public:
     MemoryManager(int num_devices, unsigned MAX_BUFFERS, bool debug)
         : mem_step_size(1024), max_buffers(MAX_BUFFERS), memory(num_devices), debug_mode(debug)
     {
         lock_guard_t lock(this->memory_mutex);
 
-        for (int n = 0; n < num_devices; n++) {
-            // Calling getMaxMemorySize() here calls the virtual function that returns 0
-            // Call it from outside the constructor.
-            memory[n].max_bytes     = ONE_GB;
-            memory[n].total_bytes   = 0;
-            memory[n].total_buffers = 0;
-            memory[n].lock_bytes    = 0;
-            memory[n].lock_buffers  = 0;
-        }
-
         // Check for environment variables
 
-        std::string env_var;
-
         // Debug mode
-        env_var = getEnvVar("AF_MEM_DEBUG");
-        if (!env_var.empty()) {
-            this->debug_mode = env_var[0] != '0';
-        }
+        std::string env_var = getEnvVar("AF_MEM_DEBUG");
+        if (!env_var.empty()) this->debug_mode = env_var[0] != '0';
         if (this->debug_mode) mem_step_size = 1;
 
         // Max Buffer count
         env_var = getEnvVar("AF_MAX_BUFFERS");
-        if (!env_var.empty()) {
-            this->max_buffers = std::max(1, std::stoi(env_var));
-        }
+        if (!env_var.empty()) this->max_buffers = std::max(1, std::stoi(env_var));
+    }
+
+    // Intended to be used with OpenCL backend, where
+    // users are allowed to add external devices(context, device pair)
+    // to the list of devices automatically detected by the library
+    void addMemoryManagement(int device)
+    {
+        // If there is a memory manager allocated for
+        // this device id, we might as well use it and the
+        // buffers allocated for it
+        if ((size_t)device < memory.size())
+            return;
+
+        // Assuming, device need not be always the next device
+        // Lets resize to current_size + device + 1
+        // +1 is to account for device being 0-based index of devices
+        memory.resize(memory.size()+device+1);
+    }
+
+    // Intended to be used with OpenCL backend, where
+    // users are allowed to add external devices(context, device pair)
+    // to the list of devices automatically detected by the library
+    void removeMemoryManagement(int device)
+    {
+        if ((size_t)device>=memory.size())
+            AF_ERROR("No matching device found", AF_ERR_ARG);
+
+        // Do garbage collection for the device and leave
+        // the memory_info struct from the memory vector intact
+        cleanDeviceMemoryManager(device);
     }
 
     void setMaxMemorySize()
@@ -230,25 +278,7 @@ class MemoryManager
 
     void garbageCollect()
     {
-        if (this->debug_mode) return;
-
-        lock_guard_t lock(this->memory_mutex);
-        memory_info& current = this->getCurrentMemoryInfo();
-
-        // Return if all buffers are locked
-        if (current.total_buffers == current.lock_buffers) return;
-
-        for (auto &kv : current.free_map) {
-            size_t num_ptrs = kv.second.size();
-            //Free memory by popping the last element
-            for (int n = num_ptrs-1; n >= 0; n--) {
-                this->nativeFree(kv.second[n]);
-                current.total_bytes -= kv.first;
-                current.total_buffers--;
-                kv.second.pop_back();
-            }
-        }
-        current.free_map.clear();
+        cleanDeviceMemoryManager(this->getActiveDeviceId());
     }
 
 
diff --git a/src/backend/opencl/platform.cpp b/src/backend/opencl/platform.cpp
index 4b43a6865a..fe6d774654 100644
--- a/src/backend/opencl/platform.cpp
+++ b/src/backend/opencl/platform.cpp
@@ -495,6 +495,9 @@ void addDeviceContext(cl_device_id dev, cl_context ctx, cl_command_queue que)
     devMngr.mPlatforms.push_back(getPlatformEnum(*tDevice));
     // FIXME: add OpenGL Interop for user provided contexts later
     devMngr.mIsGLSharingOn.push_back(false);
+
+    // Last/newly added device needs memory management
+    memoryManager().addMemoryManagement(devMngr.mDevices.size()-1);
 }
 
 void setDeviceContext(cl_device_id dev, cl_context ctx)
@@ -539,6 +542,8 @@ void removeDeviceContext(cl_device_id dev, cl_context ctx)
     } else if (deleteIdx == -1) {
         AF_ERROR("No matching device found", AF_ERR_ARG);
     } else {
+        //remove memory management for device added by user
+        memoryManager().removeMemoryManagement(deleteIdx);
 
         clReleaseDevice((*devMngr.mDevices[deleteIdx])());
         clReleaseContext((*devMngr.mContexts[deleteIdx])());
diff --git a/test/ocl_ext_context.cpp b/test/ocl_ext_context.cpp
index 3dc46991c2..13951413ee 100644
--- a/test/ocl_ext_context.cpp
+++ b/test/ocl_ext_context.cpp
@@ -55,7 +55,7 @@ void getExternals(cl_device_id &deviceId, cl_context &context, cl_command_queue
     queue    = qId;
 }
 
-TEST(OCLExtContext, push)
+TEST(OCLExtContext, PushAndPop)
 {
     cl_device_id deviceId = NULL;
     cl_context context = NULL;
@@ -63,11 +63,16 @@ TEST(OCLExtContext, push)
 
     getExternals(deviceId, context, queue);
     int dCount = af::getDeviceCount();
-    printf("%d devices before afcl::addDevice\n", dCount);
+    printf("\n%d devices before afcl::addDevice\n\n", dCount);
     af::info();
+
     afcl::addDevice(deviceId, context, queue);
     ASSERT_EQ(true, dCount+1==af::getDeviceCount());
-    printf("%d devices after afcl::addDevice\n", af::getDeviceCount());
+    printf("\n%d devices after afcl::addDevice\n", af::getDeviceCount());
+
+    afcl::deleteDevice(deviceId, context);
+    ASSERT_EQ(true, dCount==af::getDeviceCount());
+    printf("\n%d devices after afcl::deleteDevice\n\n", af::getDeviceCount());
     af::info();
 }
 
@@ -77,8 +82,19 @@ TEST(OCLExtContext, set)
     cl_context context = NULL;
     cl_command_queue queue = NULL;
 
+    int dCount = af::getDeviceCount(); //Before user device addition
+    af::setDevice(0);
+    af::info();
+    af::array t = af::randu(5,5);
+    af_print(t);
+
     getExternals(deviceId, context, queue);
-    afcl::setDevice(deviceId, context);
+    afcl::addDevice(deviceId, context, queue);
+    printf("\nBefore setting device to newly added one\n\n");
+    af::info();
+
+    printf("\n\nBefore setting device to newly added one\n\n");
+    af::setDevice(dCount); //In 0-based index, dCount is index of newly added device
     af::info();
 
     const int x = 5;
@@ -89,23 +105,11 @@ TEST(OCLExtContext, set)
     a.host((void*)host.data());
     for (int i=0; i<s; ++i)
         ASSERT_EQ(host[i], 1.0f);
-}
 
-TEST(OCLExtContext, pop)
-{
-    cl_device_id deviceId = NULL;
-    cl_context context = NULL;
-    cl_command_queue queue = NULL;
-
-    getExternals(deviceId, context, queue);
-    int dCount = af::getDeviceCount();
-    printf("%d devices before afcl::deleteDevice\n", dCount);
+    printf("\n\nAfter reset to default set of devices\n\n");
     af::setDevice(0);
     af::info();
-    afcl::deleteDevice(deviceId, context);
-    ASSERT_EQ(true, dCount-1==af::getDeviceCount());
-    printf("%d devices after afcl::deleteDevice\n", af::getDeviceCount());
-    af::info();
+    af_print(t);
 }
 
 TEST(OCLCheck, DeviceType)

From 38ef85b9283d88adecd8bbf0397440adc2c9f7df Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Fri, 24 Feb 2017 12:03:22 +0530
Subject: [PATCH 42/82] Remove shared_mutex in favour of thread_local storage

This removes the usage of boost::shared_mutex from graphics,
fft cache management and jit.

Graphcis shall be stated as thread unsafe and hence to be used
from the main thread. Rest of the caches shall use thread_local
in the subsequent commits once an alternative to thread_local
has been implemented for OSX platform.
---
 CMakeLists.txt                        |  2 +-
 src/api/c/graphics_common.cpp         | 48 -------------------
 src/backend/common/FFTPlanCache.cpp   | 69 ---------------------------
 src/backend/common/FFTPlanCache.hpp   | 27 +++++++++--
 src/backend/common/InteropManager.cpp | 20 --------
 src/backend/common/InteropManager.hpp |  1 +
 src/backend/cpu/CMakeLists.txt        |  1 -
 src/backend/cuda/CMakeLists.txt       |  1 -
 src/backend/cuda/jit.cpp              |  9 ----
 src/backend/opencl/CMakeLists.txt     |  1 -
 src/backend/opencl/platform.cpp       | 15 ------
 11 files changed, 24 insertions(+), 170 deletions(-)
 delete mode 100644 src/backend/common/FFTPlanCache.cpp

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 5d159c8ea5..6972b7d628 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -55,7 +55,7 @@ ENDIF(FREEIMAGE_FOUND)
 
 ADD_DEFINITIONS(-DBOOST_ALL_NO_LIB)
 SET(Boost_USE_STATIC_LIBS OFF)
-FIND_PACKAGE(Boost REQUIRED COMPONENTS "system" "thread")
+FIND_PACKAGE(Boost REQUIRED)
 
 OPTION(USE_SYSTEM_CL2HPP "Use cl2.hpp installed on system" OFF)
 
diff --git a/src/api/c/graphics_common.cpp b/src/api/c/graphics_common.cpp
index 88a8f8e230..249a325586 100644
--- a/src/api/c/graphics_common.cpp
+++ b/src/api/c/graphics_common.cpp
@@ -16,17 +16,10 @@
 #include <platform.hpp>
 #include <util.hpp>
 #include <mutex>
-#include <boost/thread.hpp>
 
 using namespace std;
 using namespace gl;
 
-typedef boost::shared_mutex smutex_t;
-typedef boost::shared_lock<smutex_t> rlock_t;
-typedef boost::unique_lock<smutex_t> wlock_t;
-typedef boost::upgrade_lock<smutex_t> ulock_t;
-typedef boost::upgrade_to_unique_lock<smutex_t> u2ulock_t;
-
 template<typename T>
 gl::GLenum getGLType() { return GL_FLOAT; }
 
@@ -179,13 +172,6 @@ double step_round(const double in, const bool dir)
 
 namespace graphics
 {
-static smutex_t  gImgMapMutex;
-static smutex_t  gPltMapMutex;
-static smutex_t  gHstMapMutex;
-static smutex_t  gSfcMapMutex;
-static smutex_t  gVcfMapMutex;
-static smutex_t  gChartMutex;
-
 ForgeManager& ForgeManager::getInstance()
 {
     static ForgeManager my_instance;
@@ -259,8 +245,6 @@ forge::Window* ForgeManager::getMainWindow()
 void ForgeManager::setWindowChartGrid(const forge::Window* window,
                                       const int r, const int c)
 {
-    wlock_t lock(gChartMutex);
-
     ChartMapIter iter = mChartMap.find(window);
 
     if(iter != mChartMap.end()) {
@@ -286,8 +270,6 @@ void ForgeManager::setWindowChartGrid(const forge::Window* window,
 forge::Chart* ForgeManager::getChart(const forge::Window* window, const int r, const int c,
                                      const forge::ChartType ctype)
 {
-    ulock_t lock(gChartMutex);
-
     forge::Chart* chart = NULL;
     ChartMapIter iter = mChartMap.find(window);
 
@@ -300,8 +282,6 @@ forge::Chart* ForgeManager::getChart(const forge::Window* window, const int r, c
             AF_ERROR("Grid points are out of bounds", AF_ERR_TYPE);
 
         // upgrade to exclusive access to make changes
-        u2ulock_t unqLock(lock);
-
         chart = (iter->second)[c * gRows + r];
 
         if (chart == NULL) {
@@ -340,15 +320,11 @@ forge::Image* ForgeManager::getImage(int w, int h, forge::ChannelFormat mode, fo
 
     ChartKey_t keypair = std::make_pair(key, nullptr);
 
-    ulock_t lock(gImgMapMutex);
-
     ImgMapIter iter = mImgMap.find(keypair);
 
     if (iter==mImgMap.end()) {
         forge::Image* temp = new forge::Image(w, h, mode, type);
 
-        u2ulock_t unqLock(lock);
-
         mImgMap[keypair] = temp;
     }
 
@@ -370,8 +346,6 @@ forge::Image* ForgeManager::getImage(forge::Chart* chart, int w, int h,
 
     ChartKey_t keypair = std::make_pair(key, chart);
 
-    ulock_t lock(gImgMapMutex);
-
     ImgMapIter iter = mImgMap.find(keypair);
 
     if (iter==mImgMap.end()) {
@@ -380,8 +354,6 @@ forge::Image* ForgeManager::getImage(forge::Chart* chart, int w, int h,
 
         forge::Image* temp = new forge::Image(w, h, mode, type);
 
-        u2ulock_t unqLock(lock);
-
         mImgMap[keypair] = temp;
 
         chart->add(*mImgMap[keypair]);
@@ -404,15 +376,11 @@ forge::Plot* ForgeManager::getPlot(forge::Chart* chart, int nPoints, forge::dtyp
 
     ChartKey_t keypair = std::make_pair(key, chart);
 
-    ulock_t lock(gPltMapMutex);
-
     PltMapIter iter = mPltMap.find(keypair);
 
     if (iter==mPltMap.end()) {
         forge::Plot* temp = new forge::Plot(nPoints, dtype, chart->getChartType(), ptype, mtype);
 
-        u2ulock_t unqLock(lock);
-
         mPltMap[keypair] = temp;
 
         chart->add(*mPltMap[keypair]);
@@ -433,8 +401,6 @@ forge::Histogram* ForgeManager::getHistogram(forge::Chart* chart, int nBins, for
 
     ChartKey_t keypair = std::make_pair(key, chart);
 
-    ulock_t lock(gHstMapMutex);
-
     HstMapIter iter = mHstMap.find(keypair);
 
     if (iter==mHstMap.end()) {
@@ -443,8 +409,6 @@ forge::Histogram* ForgeManager::getHistogram(forge::Chart* chart, int nBins, for
 
         forge::Histogram* temp = new forge::Histogram(nBins, type);
 
-        u2ulock_t unqLock(lock);
-
         mHstMap[keypair] = temp;
 
         chart->add(*mHstMap[keypair]);
@@ -465,8 +429,6 @@ forge::Surface* ForgeManager::getSurface(forge::Chart* chart, int nX, int nY, fo
 
     ChartKey_t keypair = std::make_pair(key, chart);
 
-    ulock_t lock(gSfcMapMutex);
-
     SfcMapIter iter = mSfcMap.find(keypair);
 
     if (iter==mSfcMap.end()) {
@@ -475,8 +437,6 @@ forge::Surface* ForgeManager::getSurface(forge::Chart* chart, int nX, int nY, fo
 
         forge::Surface* temp = new forge::Surface(nX, nY, type);
 
-        u2ulock_t unqLock(lock);
-
         mSfcMap[keypair] = temp;
 
         chart->add(*mSfcMap[keypair]);
@@ -497,15 +457,11 @@ forge::VectorField* ForgeManager::getVectorField(forge::Chart* chart, int nPoint
 
     ChartKey_t keypair = std::make_pair(key, chart);
 
-    ulock_t lock(gVcfMapMutex);
-
     VcfMapIter iter = mVcfMap.find(keypair);
 
     if (iter==mVcfMap.end()) {
         forge::VectorField* temp = new forge::VectorField(nPoints, type, chart->getChartType());
 
-        u2ulock_t unqLock(lock);
-
         mVcfMap[keypair] = temp;
 
         chart->add(*mVcfMap[keypair]);
@@ -516,8 +472,6 @@ forge::VectorField* ForgeManager::getVectorField(forge::Chart* chart, int nPoint
 
 bool ForgeManager::getChartAxesOverride(forge::Chart* chart)
 {
-    rlock_t lock(gChartMutex);
-
     ChartAxesOverrideIter iter = mChartAxesOverrideMap.find(chart);
     if (iter == mChartAxesOverrideMap.end()) {
         AF_ERROR("Chart Not Found!", AF_ERR_INTERNAL);
@@ -527,8 +481,6 @@ bool ForgeManager::getChartAxesOverride(forge::Chart* chart)
 
 void ForgeManager::setChartAxesOverride(forge::Chart* chart, bool flag)
 {
-    rlock_t lock(gChartMutex);
-
     ChartAxesOverrideIter iter = mChartAxesOverrideMap.find(chart);
     if (iter == mChartAxesOverrideMap.end()) {
         AF_ERROR("Chart Not Found!", AF_ERR_INTERNAL);
diff --git a/src/backend/common/FFTPlanCache.cpp b/src/backend/common/FFTPlanCache.cpp
deleted file mode 100644
index be5f4d9582..0000000000
--- a/src/backend/common/FFTPlanCache.cpp
+++ /dev/null
@@ -1,69 +0,0 @@
-/*******************************************************
- * Copyright (c) 2016, ArrayFire
- * All rights reserved.
- *
- * This file is distributed under 3-clause BSD license.
- * The complete license agreement can be obtained at:
- * http://arrayfire.com/licenses/BSD-3-Clause
- ********************************************************/
-
-//FIXME CPU backend doesn't required the following class implementation
-//FFTPlanCache.hpp is not used while building CPU backend.
-#ifndef AF_CPU
-#include <boost/thread.hpp>
-#include <common/FFTPlanCache.hpp>
-#include <backend.hpp>
-#include <platform.hpp>
-
-typedef boost::shared_mutex smutex_t;
-typedef boost::shared_lock<smutex_t> rlock_t;
-typedef boost::unique_lock<smutex_t> wlock_t;
-
-namespace common
-{
-static smutex_t gFFTMutexes[detail::DeviceManager::MAX_DEVICES];
-
-template<class T, typename P>
-void FFTPlanCache<T, P>::setMaxCacheSize(size_t size)
-{
-    wlock_t lock(gFFTMutexes[detail::getActiveDeviceId()]);
-    mMaxCacheSize = size;
-}
-
-template<class T, typename P>
-size_t FFTPlanCache<T, P>::getMaxCacheSize() const
-{
-    rlock_t lock(gFFTMutexes[detail::getActiveDeviceId()]);
-    return mMaxCacheSize;
-}
-
-template<class T, typename P>
-std::shared_ptr<P> FFTPlanCache<T, P>::find(const std::string& key) const
-{
-    std::shared_ptr<P> res;
-
-    rlock_t lock(gFFTMutexes[detail::getActiveDeviceId()]);
-    for(unsigned i=0; i<mCache.size(); ++i) {
-        if (key == mCache[i].first) {
-            res = mCache[i].second;
-            break;
-        }
-    }
-
-    return res;
-}
-
-template<class T, typename P>
-void FFTPlanCache<T, P>::push(const std::string key, std::shared_ptr<P> plan)
-{
-    wlock_t lock(gFFTMutexes[detail::getActiveDeviceId()]);
-
-    if (mCache.size()>=mMaxCacheSize)
-        mCache.pop_back();
-
-    mCache.push_front(plan_pair_t(key, plan));
-}
-
-template class FFTPlanCache<detail::PlanCache, detail::PlanType>;
-}
-#endif
diff --git a/src/backend/common/FFTPlanCache.hpp b/src/backend/common/FFTPlanCache.hpp
index d10baabd29..3d18f31117 100644
--- a/src/backend/common/FFTPlanCache.hpp
+++ b/src/backend/common/FFTPlanCache.hpp
@@ -8,7 +8,6 @@
  ********************************************************/
 
 #pragma once
-
 #include <deque>
 #include <memory>
 #include <string>
@@ -32,17 +31,35 @@ class FFTPlanCache
     public:
         FFTPlanCache() : mMaxCacheSize(5) {}
 
-        void setMaxCacheSize(size_t size);
-        size_t getMaxCacheSize() const;
+        void setMaxCacheSize(size_t size) { mMaxCacheSize = size; }
+        size_t getMaxCacheSize() const    { return mMaxCacheSize; }
 
         // iterates through plan cache from front to back
         // of the cache(queue)
         // A valid shared_ptr of the plan in the cache is returned
         // if found, and empty share_ptr otherwise.
-        plan_t find(const std::string& key) const;
+        plan_t find(const std::string& key) const
+        {
+            std::shared_ptr<P> res;
+
+            for(unsigned i=0; i<mCache.size(); ++i) {
+                if (key == mCache[i].first) {
+                    res = mCache[i].second;
+                    break;
+                }
+            }
+
+            return res;
+        }
 
         // pushes plan to the front of cache(queue)
-        void push(const std::string key, plan_t plan);
+        void push(const std::string key, plan_t plan)
+        {
+            if (mCache.size()>=mMaxCacheSize)
+                mCache.pop_back();
+
+            mCache.push_front(plan_pair_t(key, plan));
+        }
 
     protected:
         FFTPlanCache(FFTPlanCache const&);
diff --git a/src/backend/common/InteropManager.cpp b/src/backend/common/InteropManager.cpp
index 9f970e4324..7c1ae3bc07 100644
--- a/src/backend/common/InteropManager.cpp
+++ b/src/backend/common/InteropManager.cpp
@@ -11,7 +11,6 @@
 //FIXME CPU backend doesn't required the following class implementation
 //InteropManager.hpp is not used while building CPU backend.
 #ifndef AF_CPU
-#include <boost/thread.hpp>
 #include <common/InteropManager.hpp>
 #include <common/types.hpp>
 #include <cstdio>
@@ -21,19 +20,11 @@
 #include <GraphicsResourceManager.hpp>
 #include <platform.hpp>
 
-typedef boost::shared_mutex smutex_t;
-typedef boost::shared_lock<smutex_t> rlock_t;
-typedef boost::unique_lock<smutex_t> wlock_t;
-typedef boost::upgrade_lock<smutex_t> ulock_t;
-typedef boost::upgrade_to_unique_lock<smutex_t> u2ulock_t;
-
 template<typename R>
 using RVector = std::vector<std::shared_ptr<R>>;
 
 namespace common
 {
-static smutex_t gInteropMutexes[detail::DeviceManager::MAX_DEVICES];
-
 template<class T, typename R>
 InteropManager<T, R>::~InteropManager()
 {
@@ -51,7 +42,6 @@ InteropManager<T, R>::~InteropManager()
 template<class T, typename R>
 RVector<R> InteropManager<T, R>::getBufferResource(const forge::Image* image)
 {
-    ulock_t lock(gInteropMutexes[detail::getActiveDeviceId()]);
     void * key = (void*)image;
 
     if (mInteropMap.find(key) == mInteropMap.end()) {
@@ -59,7 +49,6 @@ RVector<R> InteropManager<T, R>::getBufferResource(const forge::Image* image)
         handles.push_back(image->pixels());
         std::vector<resource_t> output = static_cast<T*>(this)->registerResources(handles);
 
-        u2ulock_t wlock(lock);
         mInteropMap[key] = output;
     }
 
@@ -69,7 +58,6 @@ RVector<R> InteropManager<T, R>::getBufferResource(const forge::Image* image)
 template<class T, typename R>
 RVector<R> InteropManager<T, R>::getBufferResource(const forge::Plot* plot)
 {
-    ulock_t lock(gInteropMutexes[detail::getActiveDeviceId()]);
     void * key = (void*)plot;
 
     if (mInteropMap.find(key) == mInteropMap.end()) {
@@ -77,7 +65,6 @@ RVector<R> InteropManager<T, R>::getBufferResource(const forge::Plot* plot)
         handles.push_back(plot->vertices());
         std::vector<resource_t> output = static_cast<T*>(this)->registerResources(handles);
 
-        u2ulock_t wlock(lock);
         mInteropMap[key] = output;
     }
 
@@ -87,7 +74,6 @@ RVector<R> InteropManager<T, R>::getBufferResource(const forge::Plot* plot)
 template<class T, typename R>
 RVector<R> InteropManager<T, R>::getBufferResource(const forge::Histogram* histogram)
 {
-    ulock_t lock(gInteropMutexes[detail::getActiveDeviceId()]);
     void * key = (void*)histogram;
 
     if (mInteropMap.find(key) == mInteropMap.end()) {
@@ -95,7 +81,6 @@ RVector<R> InteropManager<T, R>::getBufferResource(const forge::Histogram* histo
         handles.push_back(histogram->vertices());
         std::vector<resource_t> output = static_cast<T*>(this)->registerResources(handles);
 
-        u2ulock_t wlock(lock);
         mInteropMap[key] = output;
     }
 
@@ -105,7 +90,6 @@ RVector<R> InteropManager<T, R>::getBufferResource(const forge::Histogram* histo
 template<class T, typename R>
 RVector<R> InteropManager<T, R>::getBufferResource(const forge::Surface* surface)
 {
-    ulock_t lock(gInteropMutexes[detail::getActiveDeviceId()]);
     void * key = (void*)surface;
 
     if (mInteropMap.find(key) == mInteropMap.end()) {
@@ -113,7 +97,6 @@ RVector<R> InteropManager<T, R>::getBufferResource(const forge::Surface* surface
         handles.push_back(surface->vertices());
         std::vector<resource_t> output = static_cast<T*>(this)->registerResources(handles);
 
-        u2ulock_t wlock(lock);
         mInteropMap[key] = output;
     }
 
@@ -123,7 +106,6 @@ RVector<R> InteropManager<T, R>::getBufferResource(const forge::Surface* surface
 template<class T, typename R>
 RVector<R> InteropManager<T, R>::getBufferResource(const forge::VectorField* field)
 {
-    ulock_t lock(gInteropMutexes[detail::getActiveDeviceId()]);
     void * key = (void*)field;
 
     if (mInteropMap.find(key) == mInteropMap.end()) {
@@ -132,7 +114,6 @@ RVector<R> InteropManager<T, R>::getBufferResource(const forge::VectorField* fie
         handles.push_back(field->directions());
         std::vector<resource_t> output = static_cast<T*>(this)->registerResources(handles);
 
-        u2ulock_t wlock(lock);
         mInteropMap[key] = output;
     }
 
@@ -142,7 +123,6 @@ RVector<R> InteropManager<T, R>::getBufferResource(const forge::VectorField* fie
 template<class T, typename R>
 void InteropManager<T, R>::destroyResources()
 {
-    wlock_t lock(gInteropMutexes[detail::getActiveDeviceId()]);
     for(auto iter : mInteropMap) {
         iter.second.clear();
     }
diff --git a/src/backend/common/InteropManager.hpp b/src/backend/common/InteropManager.hpp
index 5cfc3736e1..a026340afd 100644
--- a/src/backend/common/InteropManager.hpp
+++ b/src/backend/common/InteropManager.hpp
@@ -13,6 +13,7 @@
 #include <forge.h>
 #include <map>
 #include <vector>
+#include <memory>
 
 namespace common
 {
diff --git a/src/backend/cpu/CMakeLists.txt b/src/backend/cpu/CMakeLists.txt
index ab4659697f..ac94f897ea 100644
--- a/src/backend/cpu/CMakeLists.txt
+++ b/src/backend/cpu/CMakeLists.txt
@@ -214,7 +214,6 @@ TARGET_LINK_LIBRARIES(afcpu
                             PRIVATE ${CBLAS_LIBRARIES}
                             PRIVATE ${FFTW_LIBRARIES}
                             PRIVATE ${FreeImage_LIBS}
-                            PRIVATE ${Boost_LIBRARIES}
                      )
 
 IF(LAPACK_FOUND)
diff --git a/src/backend/cuda/CMakeLists.txt b/src/backend/cuda/CMakeLists.txt
index 9b3538e576..46f86dc566 100644
--- a/src/backend/cuda/CMakeLists.txt
+++ b/src/backend/cuda/CMakeLists.txt
@@ -436,7 +436,6 @@ TARGET_LINK_LIBRARIES(afcuda
                                 PRIVATE ${CUDA_cusolver_LIBRARY}
                                 PRIVATE ${CUDA_nvvm_LIBRARY}
                                 PRIVATE ${CUDA_CUDA_LIBRARY}
-                                PRIVATE ${Boost_LIBRARIES}
                      )
 
 LIST(LENGTH GRAPHICS_DEPENDENCIES GRAPHICS_DEPENDENCIES_LEN)
diff --git a/src/backend/cuda/jit.cpp b/src/backend/cuda/jit.cpp
index 1ec2e12b62..5fab3f8f5f 100644
--- a/src/backend/cuda/jit.cpp
+++ b/src/backend/cuda/jit.cpp
@@ -42,17 +42,12 @@
 #include <math.hpp>
 #include <nvvm.h>
 
-#include <boost/thread.hpp>
 #include <functional>
 #include <map>
 #include <memory>
 #include <stdexcept>
 #include <vector>
 
-typedef boost::shared_mutex smutex_t;
-typedef boost::upgrade_lock<smutex_t> ulock_t;
-typedef boost::upgrade_to_unique_lock<smutex_t> u2ulock_t;
-
 namespace cuda
 {
 
@@ -537,19 +532,15 @@ static CUfunction getKernel(vector<Node *> nodes, bool is_linear)
 {
     typedef std::map<std::string, kc_entry_t> kc_t;
 
-    static smutex_t mutexes[DeviceManager::MAX_DEVICES];
     static kc_t kernelCaches[DeviceManager::MAX_DEVICES];
 
     string funcName = getFuncName(nodes, is_linear);
     int device      = getActiveDeviceId();
 
-    ulock_t lock(mutexes[device]);
-
     kc_t::iterator idx = kernelCaches[device].find(funcName);
     kc_entry_t entry = {NULL, NULL};
 
     if (idx == kernelCaches[device].end()) {
-        u2ulock_t unqLock(lock);
         string jit_ker = getKernelString(funcName, nodes, is_linear);
         entry = compileKernel(funcName.c_str(), jit_ker);
         kernelCaches[device][funcName] = entry;
diff --git a/src/backend/opencl/CMakeLists.txt b/src/backend/opencl/CMakeLists.txt
index de2b9ec588..5a31c89405 100644
--- a/src/backend/opencl/CMakeLists.txt
+++ b/src/backend/opencl/CMakeLists.txt
@@ -319,7 +319,6 @@ TARGET_LINK_LIBRARIES(afopencl
                                 PRIVATE ${CLFFT_LIBRARIES}
                                 PRIVATE ${CMAKE_DL_LIBS}
                                 PRIVATE ${FreeImage_LIBS}
-                                PRIVATE ${Boost_LIBRARIES}
                      )
 
 IF(LAPACK_FOUND)
diff --git a/src/backend/opencl/platform.cpp b/src/backend/opencl/platform.cpp
index fe6d774654..bb0a41d82d 100644
--- a/src/backend/opencl/platform.cpp
+++ b/src/backend/opencl/platform.cpp
@@ -34,7 +34,6 @@
 #include <version.hpp>
 
 #include <algorithm>
-#include <boost/thread.hpp>
 #include <cctype>
 #include <functional>
 #include <map>
@@ -55,16 +54,8 @@ using cl::Context;
 using cl::CommandQueue;
 using cl::Device;
 
-typedef boost::shared_mutex smutex_t;
-typedef boost::shared_lock<smutex_t> rlock_t;
-typedef boost::unique_lock<smutex_t> wlock_t;
-typedef boost::upgrade_lock<smutex_t> ulock_t;
-typedef boost::upgrade_to_unique_lock<smutex_t> u2ulock_t;
-
 namespace opencl
 {
-static smutex_t kernelCacheMutexes[DeviceManager::MAX_DEVICES];
-
 #if defined (OS_MAC)
 static const std::string CL_GL_SHARING_EXT = "cl_APPLE_gl_sharing";
 #else
@@ -653,15 +644,11 @@ PlanCache& fftManager()
 
 void addKernelToCache(int device, const std::string& key, const kc_entry_t entry)
 {
-    wlock_t lock(kernelCacheMutexes[device]);
-
     DeviceManager::getInstance().kernelCaches[device].emplace(key, entry);
 }
 
 void removeKernelFromCache(int device, const std::string& key)
 {
-    wlock_t lock(kernelCacheMutexes[device]);
-
     DeviceManager::getInstance().kernelCaches[device].erase(key);
 }
 
@@ -669,8 +656,6 @@ kc_entry_t kernelCache(int device, const std::string& key)
 {
     DeviceManager& inst = DeviceManager::getInstance();
 
-    rlock_t lock(kernelCacheMutexes[device]);
-
     kc_t::iterator iter = inst.kernelCaches[device].find(key);
     if (iter == inst.kernelCaches[device].end()) {
         return kc_entry_t{0, 0};

From f80e94c9501613665a8600f8d8005a0a68d5186b Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Mon, 27 Feb 2017 23:39:03 +0530
Subject: [PATCH 43/82] Add thread_local qualifier to fft & kernel caches

---
 src/backend/cuda/jit.cpp        |  2 +-
 src/backend/cuda/platform.cpp   |  8 --------
 src/backend/opencl/platform.cpp | 29 +++++++++++++----------------
 src/backend/opencl/platform.hpp |  2 --
 4 files changed, 14 insertions(+), 27 deletions(-)

diff --git a/src/backend/cuda/jit.cpp b/src/backend/cuda/jit.cpp
index 5fab3f8f5f..f5e64551df 100644
--- a/src/backend/cuda/jit.cpp
+++ b/src/backend/cuda/jit.cpp
@@ -532,7 +532,7 @@ static CUfunction getKernel(vector<Node *> nodes, bool is_linear)
 {
     typedef std::map<std::string, kc_entry_t> kc_t;
 
-    static kc_t kernelCaches[DeviceManager::MAX_DEVICES];
+    thread_local static kc_t kernelCaches[DeviceManager::MAX_DEVICES];
 
     string funcName = getFuncName(nodes, is_linear);
     int device      = getActiveDeviceId();
diff --git a/src/backend/cuda/platform.cpp b/src/backend/cuda/platform.cpp
index 68d0cb502b..847be47c3c 100644
--- a/src/backend/cuda/platform.cpp
+++ b/src/backend/cuda/platform.cpp
@@ -421,15 +421,7 @@ GraphicsResourceManager& interopManager()
 
 PlanCache& fftManager()
 {
-    //FIXME Change to better check later, may be Clang version based check
-#if defined(OS_MAC)
-    // XCode Clang doesn't support thread_local qualifier
-    // Hence, making the cache manager per device
-    static PlanCache cufftManagers[DeviceManager::MAX_DEVICES];
-#else
-    //Otherwise, cache manager is per thread per devicea, less congestion
     thread_local static PlanCache cufftManagers[DeviceManager::MAX_DEVICES];
-#endif
 
     return cufftManagers[getActiveDeviceId()];
 }
diff --git a/src/backend/opencl/platform.cpp b/src/backend/opencl/platform.cpp
index bb0a41d82d..93690547ff 100644
--- a/src/backend/opencl/platform.cpp
+++ b/src/backend/opencl/platform.cpp
@@ -629,38 +629,35 @@ GraphicsResourceManager& interopManager()
 
 PlanCache& fftManager()
 {
-    //FIXME Change to better check later, may be Clang version based check
-#if defined(OS_MAC)
-    // XCode Clang doesn't support thread_local qualifier
-    // Hence, making the cache manager per device
-    static PlanCache clfftManagers[DeviceManager::MAX_DEVICES];
-#else
-    //Otherwise, cache manager is per thread per devicea, less congestion
     thread_local static PlanCache clfftManagers[DeviceManager::MAX_DEVICES];
-#endif
 
     return clfftManagers[getActiveDeviceId()];
 }
 
+kc_t& getKernelCache(int device)
+{
+    thread_local static kc_t kernelCaches[DeviceManager::MAX_DEVICES];
+
+    return kernelCaches[device];
+}
+
 void addKernelToCache(int device, const std::string& key, const kc_entry_t entry)
 {
-    DeviceManager::getInstance().kernelCaches[device].emplace(key, entry);
+    getKernelCache(device).emplace(key, entry);
 }
 
 void removeKernelFromCache(int device, const std::string& key)
 {
-    DeviceManager::getInstance().kernelCaches[device].erase(key);
+    getKernelCache(device).erase(key);
 }
 
 kc_entry_t kernelCache(int device, const std::string& key)
 {
-    DeviceManager& inst = DeviceManager::getInstance();
+    kc_t& cache = getKernelCache(device);
+
+    kc_t::iterator iter = cache.find(key);
 
-    kc_t::iterator iter = inst.kernelCaches[device].find(key);
-    if (iter == inst.kernelCaches[device].end()) {
-        return kc_entry_t{0, 0};
-    } else
-        return iter->second;
+    return (iter==cache.end() ? kc_entry_t{0, 0} : iter->second);
 }
 
 DeviceManager& DeviceManager::getInstance()
diff --git a/src/backend/opencl/platform.hpp b/src/backend/opencl/platform.hpp
index de55c38947..bf490d282a 100644
--- a/src/backend/opencl/platform.hpp
+++ b/src/backend/opencl/platform.hpp
@@ -194,7 +194,5 @@ class DeviceManager
 
         std::unique_ptr<GraphicsResourceManager> gfxManagers[MAX_DEVICES];
         clfftSetupData mFFTSetup;
-
-        kc_t kernelCaches[MAX_DEVICES];
 };
 }

From 8086e296cb62428af948569e4345d973a4ee5840 Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Tue, 28 Feb 2017 01:25:44 +0530
Subject: [PATCH 44/82] Fix WITH_GRAPHICS build checks in CUDA/OpenCL backends

---
 src/backend/cuda/platform.cpp   | 4 ++++
 src/backend/cuda/platform.hpp   | 8 ++++++++
 src/backend/opencl/platform.cpp | 4 ++++
 src/backend/opencl/platform.hpp | 6 ++++++
 4 files changed, 22 insertions(+)

diff --git a/src/backend/cuda/platform.cpp b/src/backend/cuda/platform.cpp
index 847be47c3c..47000273df 100644
--- a/src/backend/cuda/platform.cpp
+++ b/src/backend/cuda/platform.cpp
@@ -355,6 +355,7 @@ cudaDeviceProp getDeviceProp(int device)
 ///////////////////////////////////////////////////////////////////////////
 // DeviceManager Class Functions
 ///////////////////////////////////////////////////////////////////////////
+#if defined(WITH_GRAPHICS)
 bool DeviceManager::checkGraphicsInteropCapability()
 {
     static bool run_once = true;
@@ -377,6 +378,7 @@ bool DeviceManager::checkGraphicsInteropCapability()
 
     return capable;
 }
+#endif
 
 DeviceManager& DeviceManager::getInstance()
 {
@@ -406,6 +408,7 @@ MemoryManagerPinned& pinnedMemoryManager()
     return *(inst.pinnedMemManager.get());
 }
 
+#if defined(WITH_GRAPHICS)
 GraphicsResourceManager& interopManager()
 {
     static std::once_flag initFlags[DeviceManager::MAX_DEVICES];
@@ -418,6 +421,7 @@ GraphicsResourceManager& interopManager()
 
     return *(inst.gfxManagers[id].get());
 }
+#endif
 
 PlanCache& fftManager()
 {
diff --git a/src/backend/cuda/platform.hpp b/src/backend/cuda/platform.hpp
index f91e9da641..f3521aad21 100644
--- a/src/backend/cuda/platform.hpp
+++ b/src/backend/cuda/platform.hpp
@@ -78,7 +78,9 @@ MemoryManager& memoryManager();
 
 MemoryManagerPinned& pinnedMemoryManager();
 
+#if defined(WITH_GRAPHICS)
 GraphicsResourceManager& interopManager();
+#endif
 
 PlanCache& fftManager();
 
@@ -95,7 +97,9 @@ class DeviceManager
     public:
         static const unsigned MAX_DEVICES = 16;
 
+#if defined(WITH_GRAPHICS)
         static bool checkGraphicsInteropCapability();
+#endif
 
         static DeviceManager& getInstance();
 
@@ -103,7 +107,9 @@ class DeviceManager
 
         friend MemoryManagerPinned& pinnedMemoryManager();
 
+#if defined(WITH_GRAPHICS)
         friend GraphicsResourceManager& interopManager();
+#endif
 
         friend PlanCache& fftManager();
 
@@ -165,7 +171,9 @@ class DeviceManager
 
         std::unique_ptr<MemoryManagerPinned> pinnedMemManager;
 
+#if defined(WITH_GRAPHICS)
         std::unique_ptr<GraphicsResourceManager> gfxManagers[MAX_DEVICES];
+#endif
 
         std::unique_ptr<cublasHandle> cublasHandles[MAX_DEVICES];
 
diff --git a/src/backend/opencl/platform.cpp b/src/backend/opencl/platform.cpp
index 93690547ff..fef7afd853 100644
--- a/src/backend/opencl/platform.cpp
+++ b/src/backend/opencl/platform.cpp
@@ -614,6 +614,7 @@ MemoryManagerPinned& pinnedMemoryManager()
     return *(inst.pinnedMemManager.get());
 }
 
+#if defined(WITH_GRAPHICS)
 GraphicsResourceManager& interopManager()
 {
     static std::once_flag initFlags[DeviceManager::MAX_DEVICES];
@@ -626,6 +627,7 @@ GraphicsResourceManager& interopManager()
 
     return *(inst.gfxManagers[id].get());
 }
+#endif
 
 PlanCache& fftManager()
 {
@@ -668,9 +670,11 @@ DeviceManager& DeviceManager::getInstance()
 
 DeviceManager::~DeviceManager()
 {
+#if defined(WITH_GRAPHICS)
     for (int i=0; i<getDeviceCount(); ++i) {
         delete gfxManagers[i].release();
     }
+#endif
 #ifndef OS_WIN
     //TODO: FIXME:
     // clfftTeardown() causes a "Pure Virtual Function Called" crash on
diff --git a/src/backend/opencl/platform.hpp b/src/backend/opencl/platform.hpp
index bf490d282a..43953ec527 100644
--- a/src/backend/opencl/platform.hpp
+++ b/src/backend/opencl/platform.hpp
@@ -91,7 +91,9 @@ MemoryManager& memoryManager();
 
 MemoryManagerPinned& pinnedMemoryManager();
 
+#if defined(WITH_GRAPHICS)
 GraphicsResourceManager& interopManager();
+#endif
 
 PlanCache& fftManager();
 
@@ -109,7 +111,9 @@ class DeviceManager
 
     friend MemoryManagerPinned& pinnedMemoryManager();
 
+#if defined(WITH_GRAPHICS)
     friend GraphicsResourceManager& interopManager();
+#endif
 
     friend PlanCache& fftManager();
 
@@ -192,7 +196,9 @@ class DeviceManager
         std::unique_ptr<MemoryManager> memManager;
         std::unique_ptr<MemoryManagerPinned> pinnedMemManager;
 
+#if defined(WITH_GRAPHICS)
         std::unique_ptr<GraphicsResourceManager> gfxManagers[MAX_DEVICES];
+#endif
         clfftSetupData mFFTSetup;
 };
 }

From 124c96b0114aba1057c466c95be29864f21d5bed Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Wed, 1 Mar 2017 00:13:48 +0530
Subject: [PATCH 45/82] Support for maintaining per thread active devie Ids

This change effects CUDA/OpenCL backend.
---
 src/backend/cuda/platform.cpp   |  81 +++++++++----------
 src/backend/cuda/platform.hpp   |   4 -
 src/backend/opencl/platform.cpp | 137 ++++++++++++++++++++++++--------
 src/backend/opencl/platform.hpp |   7 --
 4 files changed, 145 insertions(+), 84 deletions(-)

diff --git a/src/backend/cuda/platform.cpp b/src/backend/cuda/platform.cpp
index 47000273df..078c1983d4 100644
--- a/src/backend/cuda/platform.cpp
+++ b/src/backend/cuda/platform.cpp
@@ -29,7 +29,6 @@ using namespace std;
 
 namespace cuda
 {
-
 ///////////////////////////////////////////////////////////////////////////
 // HELPERS
 ///////////////////////////////////////////////////////////////////////////
@@ -281,6 +280,13 @@ unsigned getMaxJitSize()
     return length;
 }
 
+int& tlocalActiveDeviceId()
+{
+    thread_local static int activeDeviceId = 0;
+
+    return activeDeviceId;
+}
+
 int getDeviceCount()
 {
     return DeviceManager::getInstance().nDevices;
@@ -288,7 +294,7 @@ int getDeviceCount()
 
 int getActiveDeviceId()
 {
-    return DeviceManager::getInstance().activeDev;
+    return tlocalActiveDeviceId();
 }
 
 int getDeviceNativeId(int device)
@@ -312,17 +318,15 @@ int getDeviceIdFromNativeId(int nativeId)
 
 cudaStream_t getStream(int device)
 {
-    cudaStream_t str = DeviceManager::getInstance().streams[device];
-    // if the stream has not yet been initialized, ie. the device has not been
-    // set to active at least once (cuz that's where the stream is created)
-    // then set the device, get the stream, reset the device to current
-    if(!str) {
-        int active_dev = DeviceManager::getInstance().activeDev;
-        setDevice(device);
-        str = DeviceManager::getInstance().streams[device];
-        setDevice(active_dev);
-    }
-    return str;
+    static std::once_flag streamInitFlags[DeviceManager::MAX_DEVICES];
+
+    std::call_once(streamInitFlags[device],
+            [device]() {
+                DeviceManager& inst = DeviceManager::getInstance();
+                CUDA_CHECK(cudaStreamCreate( & (inst.streams[device]) ));
+            });
+
+    return DeviceManager::getInstance().streams[device];
 }
 
 cudaStream_t getActiveStream()
@@ -485,7 +489,7 @@ SparseHandle sparseHandle()
 }
 
 DeviceManager::DeviceManager()
-    : cuDevices(0), activeDev(0), nDevices(0)
+    : cuDevices(0), nDevices(0)
 {
     CUDA_CHECK(cudaGetDeviceCount(&nDevices));
     if (nDevices == 0)
@@ -527,7 +531,6 @@ DeviceManager::DeviceManager()
 
 void DeviceManager::sortDevices(sort_mode mode)
 {
-    common::lock_guard_t lock(deviceMutex);
     switch(mode) {
         case memory :
             std::stable_sort(cuDevices.begin(), cuDevices.end(), card_compare_mem);
@@ -546,50 +549,49 @@ void DeviceManager::sortDevices(sort_mode mode)
 
 int DeviceManager::setActiveDevice(int device, int nId)
 {
-    static bool first = true;
-
-    common::lock_guard_t lock(deviceMutex);
+    thread_local static bool retryFlag = true;
 
     int numDevices = cuDevices.size();
 
-    if(device > numDevices) return -1;
+    if (device > numDevices)
+        return -1;
 
-    int old = activeDev;
-    if(nId == -1) nId = getDeviceNativeId(device);
-    CUDA_CHECK(cudaSetDevice(nId));
+    int old = getActiveDeviceId();
 
-    cudaError_t err = cudaSuccess;
-    if(!streams[device])
-        err = cudaStreamCreate(&streams[device]);
+    if(nId == -1)
+        nId = getDeviceNativeId(device);
 
-    activeDev = device;
+    cudaError_t err = cudaSetDevice(nId);
 
-    if (err == cudaSuccess) return old;
+    if (err == cudaSuccess) {
+        tlocalActiveDeviceId() = device;
+        return old;
+    }
 
-    // Comes when user sets device
-    // If success, return. Else throw error
-    if (!first) {
+    // For the first time a thread calls setDevice,
+    // if the requested device is unavailable, try checking
+    // for other available devices - while loop below
+    if (!retryFlag) {
         CUDA_CHECK(err);
         return old;
     }
 
-    // Comes only when first is true. Set it to false
-    first = false;
+    // Comes only when retryFlag is true. Set it to false
+    retryFlag = false;
 
     while(true) {
         // Check for errors other than DevicesUnavailable
         // If success, return. Else throw error
         // If DevicesUnavailable, try other devices (while loop below)
-        if (err != cudaErrorDevicesUnavailable) {
+        if (err != cudaErrorDeviceAlreadyInUse) {
             CUDA_CHECK(err);
-            activeDev = device;
+            tlocalActiveDeviceId() = device;
             return old;
         }
         cudaGetLastError(); // Reset error stack
 #ifndef NDEBUG
         printf("Warning: Device %d is unavailable. Incrementing to next device \n", device);
 #endif
-
         // Comes here is the device is in exclusive mode or
         // otherwise fails streamCreate with this error.
         // All other errors will error out
@@ -599,11 +601,10 @@ int DeviceManager::setActiveDevice(int device, int nId)
         // Can't call getNativeId here as it will cause an infinite loop with the constructor
         nId = cuDevices[device].nativeId;
 
-        CUDA_CHECK(cudaSetDevice(nId));
-        err = cudaStreamCreate(&streams[device]);
+        err = cudaSetDevice(nId);
     }
 
-    // If all devices fail with DevicesUnavailable, then throw this error
+    // If all devices fail with DeviceAlreadyInUse, then throw this error
     CUDA_CHECK(err);
 
     return old;
@@ -617,7 +618,8 @@ void sync(int device)
     setDevice(currDevice);
 }
 
-bool synchronize_calls() {
+bool synchronize_calls()
+{
     static bool sync = getEnvVar("AF_SYNCHRONOUS_CALLS") == "1";
     return sync;
 }
@@ -627,7 +629,6 @@ bool& evalFlag()
     static bool flag = true;
     return flag;
 }
-
 }
 
 af_err afcu_get_stream(cudaStream_t* stream, int id)
diff --git a/src/backend/cuda/platform.hpp b/src/backend/cuda/platform.hpp
index f3521aad21..152dfc6a14 100644
--- a/src/backend/cuda/platform.hpp
+++ b/src/backend/cuda/platform.hpp
@@ -131,8 +131,6 @@ class DeviceManager
 
         friend int getDeviceCount();
 
-        friend int getActiveDeviceId();
-
         friend int getDeviceNativeId(int device);
 
         friend int getDeviceIdFromNativeId(int nativeId);
@@ -154,7 +152,6 @@ class DeviceManager
         void operator=(DeviceManager const&);
 
         // Attributes
-        common::mutex_t deviceMutex;
         std::vector<cudaDevice_t> cuDevices;
 
         enum sort_mode {flops = 0, memory = 1, compute = 2, none = 3};
@@ -163,7 +160,6 @@ class DeviceManager
 
         int setActiveDevice(int device, int native = -1);
 
-        int activeDev;
         int nDevices;
         cudaStream_t streams[MAX_DEVICES];
 
diff --git a/src/backend/opencl/platform.cpp b/src/backend/opencl/platform.cpp
index fef7afd853..3ea48f4ac6 100644
--- a/src/backend/opencl/platform.cpp
+++ b/src/backend/opencl/platform.cpp
@@ -42,6 +42,7 @@
 #include <sstream>
 #include <stdexcept>
 #include <cstring>
+#include <utility>
 #include <vector>
 
 using std::string;
@@ -222,12 +223,16 @@ static std::string platformMap(std::string &platStr)
 
 std::string getDeviceInfo()
 {
+    DeviceManager& devMngr = DeviceManager::getInstance();
+
+    common::lock_guard_t lock(devMngr.deviceMutex);
+
     ostringstream info;
     info << "ArrayFire v" << AF_VERSION
          << " (OpenCL, " << get_system() << ", build " << AF_REVISION << ")" << std::endl;
 
     unsigned nDevices = 0;
-    for(auto &device: DeviceManager::getInstance().mDevices) {
+    for(auto &device: devMngr.mDevices) {
         const Platform platform(device->getInfo<CL_DEVICE_PLATFORM>());
 
         string dstr = device->getInfo<CL_DEVICE_NAME>();
@@ -267,60 +272,117 @@ std::string getPlatformName(const cl::Device &device)
     return platformMap(platStr);
 }
 
+typedef std::pair<unsigned, unsigned> device_id_t;
+
+std::pair<unsigned, unsigned>& tlocalActiveDeviceId()
+{
+    // First element is active context id
+    // Second element is active queue id
+    thread_local static device_id_t activeDeviceId(0, 0);
+
+    return activeDeviceId;
+}
+
+void setActiveContext(int device)
+{
+    tlocalActiveDeviceId() = std::make_pair(device, device);
+}
+
 int getDeviceCount()
 {
-    return DeviceManager::getInstance().mQueues.size();
+    DeviceManager& devMngr = DeviceManager::getInstance();
+
+    common::lock_guard_t lock(devMngr.deviceMutex);
+
+    return devMngr.mQueues.size();
 }
 
 int getActiveDeviceId()
 {
-    return DeviceManager::getInstance().mActiveQId;
+    // Second element is the queue id, which is
+    // what we mean by active device id in opencl backend
+    return std::get<1>(tlocalActiveDeviceId());
 }
 
 int getDeviceIdFromNativeId(cl_device_id id)
 {
     DeviceManager& devMngr = DeviceManager::getInstance();
+
+    common::lock_guard_t lock(devMngr.deviceMutex);
+
     int nDevices = devMngr.mDevices.size();
     int devId = 0;
     for (devId=0; devId<nDevices; ++devId) {
         if (id == devMngr.mDevices[devId]->operator()())
             break;
     }
+
     return devId;
 }
 
 int getActiveDeviceType()
 {
-    DeviceManager &instance = DeviceManager::getInstance();
-    return instance.mDeviceTypes[instance.mActiveQId];
+    device_id_t& devId = tlocalActiveDeviceId();
+
+    DeviceManager& devMngr = DeviceManager::getInstance();
+
+    common::lock_guard_t lock(devMngr.deviceMutex);
+
+    return devMngr.mDeviceTypes[std::get<1>(devId)];
 }
 
 int getActivePlatform()
 {
-    DeviceManager &instance = DeviceManager::getInstance();
-    return instance.mPlatforms[instance.mActiveQId];
+    device_id_t& devId = tlocalActiveDeviceId();
+
+    DeviceManager& devMngr = DeviceManager::getInstance();
+
+    common::lock_guard_t lock(devMngr.deviceMutex);
+
+    return devMngr.mPlatforms[std::get<1>(devId)];
 }
 const Context& getContext()
 {
+    device_id_t& devId = tlocalActiveDeviceId();
+
     DeviceManager& devMngr = DeviceManager::getInstance();
-    return *(devMngr.mContexts[devMngr.mActiveCtxId]);
+
+    common::lock_guard_t lock(devMngr.deviceMutex);
+
+    return *(devMngr.mContexts[std::get<0>(devId)]);
 }
 
 CommandQueue& getQueue()
 {
+    device_id_t& devId = tlocalActiveDeviceId();
+
     DeviceManager& devMngr = DeviceManager::getInstance();
-    return *(devMngr.mQueues[devMngr.mActiveQId]);
+
+    common::lock_guard_t lock(devMngr.deviceMutex);
+
+    return *(devMngr.mQueues[std::get<1>(devId)]);
 }
 
 const cl::Device& getDevice(int id)
 {
+    device_id_t& devId = tlocalActiveDeviceId();
+
+    if (id == -1)
+        id = std::get<1>(devId);
+
     DeviceManager& devMngr = DeviceManager::getInstance();
-    if(id == -1) id = devMngr.mActiveQId;
+
+    common::lock_guard_t lock(devMngr.deviceMutex);
+
     return *(devMngr.mDevices[id]);
 }
 
 size_t getDeviceMemorySize(int device)
 {
+    DeviceManager& devMngr = DeviceManager::getInstance();
+
+    common::lock_guard_t lock(devMngr.deviceMutex);
+
     const cl::Device& dev = getDevice(device);
     size_t msize = dev.getInfo<CL_DEVICE_GLOBAL_MEM_SIZE>();
     return msize;
@@ -368,13 +430,21 @@ bool OpenCLCPUOffload(bool forceOffloadOSX)
 
 bool isGLSharingSupported()
 {
+    device_id_t& devId = tlocalActiveDeviceId();
+
     DeviceManager& devMngr = DeviceManager::getInstance();
-    return devMngr.mIsGLSharingOn[devMngr.mActiveQId];
+
+    common::lock_guard_t lock(devMngr.deviceMutex);
+
+    return devMngr.mIsGLSharingOn[std::get<1>(devId)];
 }
 
 bool isDoubleSupported(int device)
 {
     DeviceManager& devMngr = DeviceManager::getInstance();
+
+    common::lock_guard_t lock(devMngr.deviceMutex);
+
     return (devMngr.mDevices[device]->getInfo<CL_DEVICE_PREFERRED_VECTOR_WIDTH_DOUBLE>()>0);
 }
 
@@ -384,7 +454,11 @@ void devprop(char* d_name, char* d_platform, char *d_toolkit, char* d_compute)
     unsigned currActiveDevId = (unsigned)getActiveDeviceId();
     bool devset = false;
 
-    for (auto context : DeviceManager::getInstance().mContexts) {
+    DeviceManager& devMngr = DeviceManager::getInstance();
+
+    common::lock_guard_t lock(devMngr.deviceMutex);
+
+    for (auto context : devMngr.mContexts) {
         vector<Device> devices = context->getInfo<CL_CONTEXT_DEVICES>();
 
         for (auto &device : devices) {
@@ -430,14 +504,15 @@ int setDevice(int device)
 {
     DeviceManager& devMngr = DeviceManager::getInstance();
 
+    common::lock_guard_t lock(devMngr.deviceMutex);
+
     if (device >= (int)devMngr.mQueues.size() ||
             device>= (int)DeviceManager::MAX_DEVICES) {
         //throw runtime_error("@setDevice: invalid device index");
         return -1;
-    }
-    else {
-        int old = devMngr.mActiveQId;
-        devMngr.setContext(device);
+    } else {
+        int old = getActiveDeviceId();
+        setActiveContext(device);
         return old;
     }
 }
@@ -551,17 +626,22 @@ void removeDeviceContext(cl_device_id dev, cl_context ctx)
         devMngr.mContexts.erase(devMngr.mContexts.begin()+deleteIdx);
         devMngr.mQueues.erase(devMngr.mQueues.begin()+deleteIdx);
         devMngr.mPlatforms.erase(devMngr.mPlatforms.begin()+deleteIdx);
+
         // FIXME: add OpenGL Interop for user provided contexts later
         devMngr.mIsGLSharingOn.erase(devMngr.mIsGLSharingOn.begin()+deleteIdx);
-        // OTHERWISE, update(decrement) the `mActive*Id` variables
-        if (deleteIdx < (int)devMngr.mActiveCtxId) {
-            --devMngr.mActiveCtxId;
-            --devMngr.mActiveQId;
+
+        // OTHERWISE, update(decrement) the thread local active device ids
+        device_id_t& devId = tlocalActiveDeviceId();
+
+        if (deleteIdx < (int)devId.first) {
+            device_id_t newVals = std::make_pair(devId.first-1, devId.second-1);
+            devId = newVals;
         }
     }
 }
 
-bool synchronize_calls() {
+bool synchronize_calls()
+{
     static bool sync = getEnvVar("AF_SYNCHRONOUS_CALLS") == "1";
     return sync;
 }
@@ -698,16 +778,8 @@ DeviceManager::~DeviceManager()
 #endif
 }
 
-void DeviceManager::setContext(int device)
-{
-    common::lock_guard_t lock(deviceMutex);
-
-    mActiveQId = device;
-    mActiveCtxId = device;
-}
-
 DeviceManager::DeviceManager()
-    : mUserDeviceOffset(0), mActiveCtxId(0), mActiveQId(0)
+    : mUserDeviceOffset(0)
 {
     std::vector<cl::Platform>   platforms;
     Platform::get(&platforms);
@@ -781,7 +853,7 @@ DeviceManager::DeviceManager()
             printf("WARNING: AF_OPENCL_DEFAULT_DEVICE is out of range\n");
             printf("Setting default device as 0\n");
         } else {
-            setContext(def_device);
+            setActiveContext(def_device);
             default_device_set = true;
         }
     }
@@ -800,7 +872,7 @@ DeviceManager::DeviceManager()
         for (int i = 0; i < nDevices; i++) {
             if (mDevices[i]->getInfo<CL_DEVICE_TYPE>() == default_device_type) {
                 default_device_set = true;
-                setContext(i);
+                setActiveContext(i);
                 break;
             }
         }
@@ -836,7 +908,6 @@ DeviceManager::DeviceManager()
 #if defined(WITH_GRAPHICS)
 void DeviceManager::markDeviceForInterop(const int device, const forge::Window* wHandle)
 {
-    common::lock_guard_t lock(deviceMutex);
     try {
         if (device >= (int)mQueues.size() ||
                 device>= (int)DeviceManager::MAX_DEVICES) {
diff --git a/src/backend/opencl/platform.hpp b/src/backend/opencl/platform.hpp
index 43953ec527..1d132ca820 100644
--- a/src/backend/opencl/platform.hpp
+++ b/src/backend/opencl/platform.hpp
@@ -127,8 +127,6 @@ class DeviceManager
 
     friend int getDeviceCount();
 
-    friend int getActiveDeviceId();
-
     friend int getDeviceIdFromNativeId(cl_device_id id);
 
     friend const cl::Context& getContext();
@@ -165,8 +163,6 @@ class DeviceManager
         ~DeviceManager();
 
     protected:
-        void setContext(int device);
-
         DeviceManager();
 
         // Following two declarations are required to
@@ -190,9 +186,6 @@ class DeviceManager
         std::vector<int>         mPlatforms;
         unsigned mUserDeviceOffset;
 
-        unsigned mActiveCtxId;
-        unsigned mActiveQId;
-
         std::unique_ptr<MemoryManager> memManager;
         std::unique_ptr<MemoryManagerPinned> pinnedMemManager;
 

From 86b715c9e74473a364491fa36b16f63637abb23b Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Fri, 3 Mar 2017 09:48:36 +0530
Subject: [PATCH 46/82] Add basic test that validates morph function results
 from 2 threads

---
 test/CMakeLists.txt |   2 -
 test/threading.cpp  | 141 ++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 141 insertions(+), 2 deletions(-)
 create mode 100644 test/threading.cpp

diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 65b91855e4..b1023d1e14 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -52,8 +52,6 @@ ELSE()
     ENABLE_TESTING()
 ENDIF()
 
-REMOVE_DEFINITIONS(-std=c++11)
-
 MACRO(CREATE_TESTS BACKEND AFLIBNAME GTEST_LIBS OTHER_LIBS)
     STRING(TOUPPER ${BACKEND} DEF_NAME)
 
diff --git a/test/threading.cpp b/test/threading.cpp
new file mode 100644
index 0000000000..e64337c731
--- /dev/null
+++ b/test/threading.cpp
@@ -0,0 +1,141 @@
+/*******************************************************
+ * Copyright (c) 2017, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <cstddef>
+#include <gtest/gtest.h>
+#include <arrayfire.h>
+#include <testHelpers.hpp>
+#include <thread>
+#include <chrono>
+
+using namespace af;
+
+using std::vector;
+using std::string;
+
+#if defined(AF_CPU)
+static const unsigned ITERATION_COUNT = 10;
+#else
+static const unsigned ITERATION_COUNT = 1000;
+#endif
+
+void morphTest(const array input, const array mask, const bool isDilation,
+               const array gold, int targetDevice)
+{
+    auto start = std::chrono::high_resolution_clock::now();
+
+    af::setDevice(targetDevice);
+
+    vector<float> goldData(gold.elements());
+    vector<float> outData(gold.elements());
+
+    gold.host((void*)goldData.data());
+
+    af::array out;
+
+    for (unsigned i=0; i<ITERATION_COUNT; ++i)
+        out = isDilation ? dilate(input, mask) : erode(input, mask);
+
+    out.host((void*)outData.data());
+
+    ASSERT_EQ(true, compareArraysRMSD(gold.elements(), goldData.data(), outData.data(), 0.018f));
+
+    auto end = std::chrono::high_resolution_clock::now();
+
+    std::chrono::duration<double> diff = end - start;
+
+    std::cout << "Thread(" << std::this_thread::get_id()
+              << "): time taken for "
+              << ITERATION_COUNT
+              <<" is "
+              << diff.count() << " s\n";
+}
+
+TEST(Threading, SimultaneousRead)
+{
+    if (noImageIOTests()) return;
+
+    vector<bool> isDilationFlags;
+    vector<bool> isColorFlags;
+    vector<string> files;
+
+    files.push_back( string(TEST_DIR "/morph/gray.test") );
+    isDilationFlags.push_back(true);
+    isColorFlags.push_back(false);
+
+    files.push_back( string(TEST_DIR "/morph/color.test") );
+    isDilationFlags.push_back(false);
+    isColorFlags.push_back(true);
+
+    vector<std::thread> tests;
+    unsigned totalTestCount = 0;
+
+    auto start = std::chrono::high_resolution_clock::now();
+
+    for(size_t pos = 0; pos<files.size(); ++pos)
+    {
+        const bool isDilation = isDilationFlags[pos];
+        const bool isColor    = isColorFlags[pos];
+
+        vector<dim4>    inDims;
+        vector<string>  inFiles;
+        vector<dim_t>   outSizes;
+        vector<string>  outFiles;
+
+        readImageTests(files[pos], inDims, inFiles, outSizes, outFiles);
+
+        const unsigned testCount = inDims.size();
+
+        const dim4 maskdims(3,3,1,1);
+
+        for (size_t testId=0; testId<testCount; ++testId)
+        {
+            int trgtDeviceId = totalTestCount % af::getDeviceCount();
+
+            //prefix full path to image file names
+            inFiles[testId].insert(0,string(TEST_DIR "/morph/"));
+            outFiles[testId].insert(0,string(TEST_DIR "/morph/"));
+
+            af::setDevice(trgtDeviceId);
+
+            const array mask = constant(1.0, maskdims);
+
+            array input= loadImage(inFiles[testId].c_str(), isColor);
+            array gold = loadImage(outFiles[testId].c_str(), isColor);
+
+            //Push the new test as a new thread of execution
+            tests.emplace_back(morphTest, input, mask, isDilation, gold, trgtDeviceId);
+
+            std::cout<<"morph test launched with the following params on device ("
+                     <<trgtDeviceId<<"):"<<std::endl;
+            std::cout<<"\t Input image dims: "<<input.dims()<<std::endl;
+            std::cout<<"\t Mask dims: "<<mask.dims()<<std::endl;
+            std::cout<<"\t IsDilation : "<< (isDilation ? "True" : "False") <<std::endl;
+
+            totalTestCount++;
+        }
+    }
+    std::cout<< std::endl << "Waiting for results ..." << std::endl << std::endl;
+
+    for (size_t testId=0; testId<tests.size(); ++testId)
+    {
+        if (tests[testId].joinable()) {
+            std::cout<<"Attempting join for test ..."<<testId<<std::endl;
+            tests[testId].join();
+            std::cout<<"test "<< testId <<" completed." << std::endl;
+        }
+        std::cout<<std::endl;
+    }
+
+    auto end = std::chrono::high_resolution_clock::now();
+
+    std::chrono::duration<double> diff = end - start;
+
+    std::cout << "Total time taken for test : " << diff.count() << " s\n";
+}

From b5aea95552ee0363d474df1e53e5787b75349ded Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Sat, 4 Mar 2017 09:20:12 +0530
Subject: [PATCH 47/82] Threading test with 32 threads calling arithmetic JIT
 operations

---
 test/threading.cpp | 192 +++++++++++++++++++++++++++++----------------
 1 file changed, 123 insertions(+), 69 deletions(-)

diff --git a/test/threading.cpp b/test/threading.cpp
index e64337c731..a1e084f227 100644
--- a/test/threading.cpp
+++ b/test/threading.cpp
@@ -57,85 +57,139 @@ void morphTest(const array input, const array mask, const bool isDilation,
               << diff.count() << " s\n";
 }
 
-TEST(Threading, SimultaneousRead)
+//TEST(Threading, SetPerThreadActiveDevice)
+//{
+//    if (noImageIOTests()) return;
+//
+//    vector<bool> isDilationFlags;
+//    vector<bool> isColorFlags;
+//    vector<string> files;
+//
+//    files.push_back( string(TEST_DIR "/morph/gray.test") );
+//    isDilationFlags.push_back(true);
+//    isColorFlags.push_back(false);
+//
+//    files.push_back( string(TEST_DIR "/morph/color.test") );
+//    isDilationFlags.push_back(false);
+//    isColorFlags.push_back(true);
+//
+//    vector<std::thread> tests;
+//    unsigned totalTestCount = 0;
+//
+//    auto start = std::chrono::high_resolution_clock::now();
+//
+//    for(size_t pos = 0; pos<files.size(); ++pos)
+//    {
+//        const bool isDilation = isDilationFlags[pos];
+//        const bool isColor    = isColorFlags[pos];
+//
+//        vector<dim4>    inDims;
+//        vector<string>  inFiles;
+//        vector<dim_t>   outSizes;
+//        vector<string>  outFiles;
+//
+//        readImageTests(files[pos], inDims, inFiles, outSizes, outFiles);
+//
+//        const unsigned testCount = inDims.size();
+//
+//        const dim4 maskdims(3,3,1,1);
+//
+//        for (size_t testId=0; testId<testCount; ++testId)
+//        {
+//            int trgtDeviceId = totalTestCount % af::getDeviceCount();
+//
+//            //prefix full path to image file names
+//            inFiles[testId].insert(0,string(TEST_DIR "/morph/"));
+//            outFiles[testId].insert(0,string(TEST_DIR "/morph/"));
+//
+//            af::setDevice(trgtDeviceId);
+//
+//            const array mask = constant(1.0, maskdims);
+//
+//            array input= loadImage(inFiles[testId].c_str(), isColor);
+//            array gold = loadImage(outFiles[testId].c_str(), isColor);
+//
+//            //Push the new test as a new thread of execution
+//            tests.emplace_back(morphTest, input, mask, isDilation, gold, trgtDeviceId);
+//
+//            std::cout<<"morph test launched with the following params on device ("
+//                     <<trgtDeviceId<<"):"<<std::endl;
+//            std::cout<<"\t Input image dims: "<<input.dims()<<std::endl;
+//            std::cout<<"\t Mask dims: "<<mask.dims()<<std::endl;
+//            std::cout<<"\t IsDilation : "<< (isDilation ? "True" : "False") <<std::endl;
+//
+//            totalTestCount++;
+//        }
+//    }
+//    std::cout<< std::endl << "Waiting for results ..." << std::endl << std::endl;
+//
+//    for (size_t testId=0; testId<tests.size(); ++testId)
+//    {
+//        if (tests[testId].joinable()) {
+//            std::cout<<"Attempting join for test ..."<<testId<<std::endl;
+//            tests[testId].join();
+//            std::cout<<"test "<< testId <<" completed." << std::endl;
+//        }
+//        std::cout<<std::endl;
+//    }
+//
+//    auto end = std::chrono::high_resolution_clock::now();
+//
+//    std::chrono::duration<double> diff = end - start;
+//
+//    std::cout << "Total time taken for test : " << diff.count() << " s\n";
+//}
+
+enum ArithOp
 {
-    if (noImageIOTests()) return;
-
-    vector<bool> isDilationFlags;
-    vector<bool> isColorFlags;
-    vector<string> files;
-
-    files.push_back( string(TEST_DIR "/morph/gray.test") );
-    isDilationFlags.push_back(true);
-    isColorFlags.push_back(false);
-
-    files.push_back( string(TEST_DIR "/morph/color.test") );
-    isDilationFlags.push_back(false);
-    isColorFlags.push_back(true);
-
-    vector<std::thread> tests;
-    unsigned totalTestCount = 0;
-
-    auto start = std::chrono::high_resolution_clock::now();
+    ADD, SUB, DIV, MUL
+};
 
-    for(size_t pos = 0; pos<files.size(); ++pos)
+void calc(ArithOp opcode, array op1, array op2, float outValue)
+{
+    af::setDevice(0);
+    array res;
+    for (unsigned i=0; i<ITERATION_COUNT; ++i)
     {
-        const bool isDilation = isDilationFlags[pos];
-        const bool isColor    = isColorFlags[pos];
-
-        vector<dim4>    inDims;
-        vector<string>  inFiles;
-        vector<dim_t>   outSizes;
-        vector<string>  outFiles;
-
-        readImageTests(files[pos], inDims, inFiles, outSizes, outFiles);
-
-        const unsigned testCount = inDims.size();
-
-        const dim4 maskdims(3,3,1,1);
-
-        for (size_t testId=0; testId<testCount; ++testId)
-        {
-            int trgtDeviceId = totalTestCount % af::getDeviceCount();
-
-            //prefix full path to image file names
-            inFiles[testId].insert(0,string(TEST_DIR "/morph/"));
-            outFiles[testId].insert(0,string(TEST_DIR "/morph/"));
-
-            af::setDevice(trgtDeviceId);
-
-            const array mask = constant(1.0, maskdims);
+        switch(opcode) {
+            case ADD: res = op1 + op2; break;
+            case SUB: res = op1 - op2; break;
+            case DIV: res = op1 / op2; break;
+            case MUL: res = op1 * op2; break;
+        }
+    }
 
-            array input= loadImage(inFiles[testId].c_str(), isColor);
-            array gold = loadImage(outFiles[testId].c_str(), isColor);
+    std::vector<float> out(res.elements());
+    res.host((void*)out.data());
 
-            //Push the new test as a new thread of execution
-            tests.emplace_back(morphTest, input, mask, isDilation, gold, trgtDeviceId);
+    for (unsigned i=0; i <out.size(); ++i)
+        ASSERT_EQ(out[i], outValue);
+}
 
-            std::cout<<"morph test launched with the following params on device ("
-                     <<trgtDeviceId<<"):"<<std::endl;
-            std::cout<<"\t Input image dims: "<<input.dims()<<std::endl;
-            std::cout<<"\t Mask dims: "<<mask.dims()<<std::endl;
-            std::cout<<"\t IsDilation : "<< (isDilation ? "True" : "False") <<std::endl;
+TEST(Threading, SimultaneousRead)
+{
+    af::setDevice(0);
+    af::array A = af::constant(1.0, 100, 100);
+    af::array B = af::constant(1.0, 100, 100);
 
-            totalTestCount++;
-        }
-    }
-    std::cout<< std::endl << "Waiting for results ..." << std::endl << std::endl;
+    vector<std::thread> tests;
 
-    for (size_t testId=0; testId<tests.size(); ++testId)
+    for (int t=0; t<32; ++t)
     {
-        if (tests[testId].joinable()) {
-            std::cout<<"Attempting join for test ..."<<testId<<std::endl;
-            tests[testId].join();
-            std::cout<<"test "<< testId <<" completed." << std::endl;
+        ArithOp op;
+        float outValue;
+
+        switch(t%4) {
+            case 0: op = ADD; outValue = 2.0f; break;
+            case 1: op = SUB; outValue = 0.0f; break;
+            case 2: op = DIV; outValue = 1.0f; break;
+            case 3: op = MUL; outValue = 1.0f; break;
         }
-        std::cout<<std::endl;
-    }
 
-    auto end = std::chrono::high_resolution_clock::now();
-
-    std::chrono::duration<double> diff = end - start;
+        tests.emplace_back(calc, op, A, B, outValue);
+    }
 
-    std::cout << "Total time taken for test : " << diff.count() << " s\n";
+    for (int t=0; t<32; ++t)
+        if (tests[t].joinable())
+            tests[t].join();
 }

From 2ce12436c2ae734da0a447308ceef204ee332419 Mon Sep 17 00:00:00 2001
From: Pavan Yalamanchili <contact@pavanky.com>
Date: Mon, 13 Mar 2017 23:41:24 -0700
Subject: [PATCH 48/82] Changes required to make JIT in CUDA backend thread
 safe

---
 src/backend/cuda/Array.cpp          |  28 +++++-
 src/backend/cuda/Array.hpp          |  17 +---
 src/backend/cuda/JIT/BinaryNode.hpp | 132 +++++--------------------
 src/backend/cuda/JIT/BufferNode.hpp | 147 +++++++++++-----------------
 src/backend/cuda/JIT/Node.hpp       |  98 ++++++++-----------
 src/backend/cuda/JIT/ScalarNode.hpp |  54 +---------
 src/backend/cuda/JIT/UnaryNode.hpp  | 117 ++++------------------
 src/backend/cuda/Param.hpp          |  16 ++-
 src/backend/cuda/jit.cpp            | 103 ++++++++++++-------
 9 files changed, 261 insertions(+), 451 deletions(-)

diff --git a/src/backend/cuda/Array.cpp b/src/backend/cuda/Array.cpp
index d34abd1d42..495f35807e 100644
--- a/src/backend/cuda/Array.cpp
+++ b/src/backend/cuda/Array.cpp
@@ -240,8 +240,13 @@ namespace cuda
 
                     unsigned length =0, buf_count = 0, bytes = 0;
                     Node *n = node.get();
-                    n->getInfo(length, buf_count, bytes);
-                    n->resetFlags();
+                    JIT::Node_map_t nodes_map;
+                    n->getNodesMap(nodes_map);
+
+                    for(auto &entry : nodes_map) {
+                        Node *node = entry.first;
+                        node->getInfo(length, buf_count, bytes);
+                    }
 
                     if (2 * bytes > lock_bytes) {
                         out.eval();
@@ -360,6 +365,18 @@ namespace cuda
         return;
     }
 
+    template<typename T>
+    void
+    Array<T>::setDataDims(const dim4 &new_dims)
+    {
+        modDims(new_dims);
+        data_dims = new_dims;
+        if (node->isBuffer()) {
+            node = bufferNodePtr<T>();
+        }
+    }
+
+
 #define INSTANTIATE(T)                                                  \
     template       Array<T>  createHostDataArray<T>   (const dim4 &size, const T * const data); \
     template       Array<T>  createDeviceDataArray<T> (const dim4 &size, const void *data); \
@@ -382,9 +399,12 @@ namespace cuda
     template       void Array<T>::eval();                               \
     template       void Array<T>::eval() const;                         \
     template       T*   Array<T>::device();                             \
-    template       void      writeHostDataArray<T>    (Array<T> &arr, const T * const data, const size_t bytes); \
-    template       void      writeDeviceDataArray<T>  (Array<T> &arr, const void * const data, const size_t bytes); \
+    template       void      writeHostDataArray<T>    (Array<T> &arr, const T * const data, \
+                                                       const size_t bytes); \
+    template       void      writeDeviceDataArray<T>  (Array<T> &arr, const void * const data, \
+                                                       const size_t bytes); \
     template       void      evalMultiple<T>     (std::vector<Array<T>*> arrays); \
+    template       void Array<T>::setDataDims(const dim4 &new_dims);    \
 
     INSTANTIATE(float)
     INSTANTIATE(double)
diff --git a/src/backend/cuda/Array.hpp b/src/backend/cuda/Array.hpp
index b7092ad867..1136cdaea4 100644
--- a/src/backend/cuda/Array.hpp
+++ b/src/backend/cuda/Array.hpp
@@ -169,11 +169,7 @@ namespace cuda
             return data_dims;
         }
 
-        void setDataDims(const dim4 &new_dims)
-        {
-            modDims(new_dims);
-            data_dims = new_dims;
-        }
+        void setDataDims(const dim4 &new_dims);
 
         size_t getAllocatedBytes() const
         {
@@ -208,19 +204,12 @@ namespace cuda
 
         operator Param<T>()
         {
-            Param<T> out;
-            out.ptr = this->get();
-            for (int  i = 0; i < 4; i++) {
-                out.dims[i] = dims()[i];
-                out.strides[i] = strides()[i];
-            }
-            return out;
+            return Param<T>(this->get(), this->dims().get(), this->strides().get());
         }
 
         operator CParam<T>() const
         {
-            CParam<T> out(this->get(), this->dims().get(), this->strides().get());
-            return out;
+            return CParam<T>(this->get(), this->dims().get(), this->strides().get());
         }
 
         JIT::Node_ptr getNode();
diff --git a/src/backend/cuda/JIT/BinaryNode.hpp b/src/backend/cuda/JIT/BinaryNode.hpp
index a07d83d111..115c892119 100644
--- a/src/backend/cuda/JIT/BinaryNode.hpp
+++ b/src/backend/cuda/JIT/BinaryNode.hpp
@@ -20,148 +20,68 @@ namespace JIT
     class BinaryNode : public Node
     {
     private:
-        std::string m_op_str;
-        Node_ptr m_lhs, m_rhs;
-        int m_op;
-        int m_call_type;
+        const std::string m_op_str;
+        const int m_op;
+        const int m_call_type;
 
     public:
         BinaryNode(const char *out_type_str, const char *name_str,
                    const std::string &op_str,
                    Node_ptr lhs, Node_ptr rhs, int op, int call_type)
-            : Node(out_type_str, name_str, std::max(lhs->getHeight(), rhs->getHeight()) + 1),
+            : Node(out_type_str, name_str, std::max(lhs->getHeight(), rhs->getHeight()) + 1, {lhs, rhs}),
               m_op_str(op_str),
-              m_lhs(lhs),
-              m_rhs(rhs),
               m_op(op),
               m_call_type(call_type)
         {
         }
 
-        bool isLinear(dim_t dims[4])
+        void genKerName(std::stringstream &kerStream, Node_ids ids)
         {
-            if (!m_set_is_linear) {
-                m_linear = m_lhs->isLinear(dims) && m_rhs->isLinear(dims);
-                m_set_is_linear = true;
-            }
-            return m_linear;
-        }
-
-        void genParams(std::stringstream &kerStream,
-                       std::stringstream &annStream, bool is_linear)
-        {
-            if (m_gen_param) return;
-            if (!(m_lhs->isGenParam())) m_lhs->genParams(kerStream, annStream, is_linear);
-            if (!(m_rhs->isGenParam())) m_rhs->genParams(kerStream, annStream, is_linear);
-            m_gen_param = true;
-        }
-
-        void genOffsets(std::stringstream &kerStream, bool is_linear)
-        {
-            if (m_gen_offset) return;
-            if (!(m_lhs->isGenOffset())) m_lhs->genOffsets(kerStream, is_linear);
-            if (!(m_rhs->isGenOffset())) m_rhs->genOffsets(kerStream, is_linear);
-            m_gen_offset = true;
-        }
-
-        void genKerName(std::stringstream &kerStream)
-        {
-            if (m_gen_name) return;
-            m_lhs->genKerName(kerStream);
-            m_rhs->genKerName(kerStream);
-
             // Make the hex representation of enum part of the Kernel name
             kerStream << "_" << std::setw(2) << std::setfill('0') << std::hex << m_op;
-            kerStream << std::setw(2) << std::setfill('0') << std::hex << m_lhs->getId();
-            kerStream << std::setw(2) << std::setfill('0') << std::hex << m_rhs->getId();
-            kerStream << std::setw(2) << std::setfill('0') << std::hex << m_id << std::dec;
-            m_gen_name = true;
+            kerStream << std::setw(2) << std::setfill('0') << std::hex << ids.child_ids[0];
+            kerStream << std::setw(2) << std::setfill('0') << std::hex << ids.child_ids[1];
+            kerStream << std::setw(2) << std::setfill('0') << std::hex << ids.id << std::dec;
         }
 
-        void genFuncs(std::stringstream &kerStream, str_map_t &declStrs, bool is_linear)
+        void genFuncs(std::stringstream &kerStream, str_map_t &declStrs, Node_ids ids, bool is_linear)
         {
-            if (m_gen_func) return;
-
-            if (!(m_lhs->isGenFunc())) m_lhs->genFuncs(kerStream, declStrs, is_linear);
-            if (!(m_rhs->isGenFunc())) m_rhs->genFuncs(kerStream, declStrs, is_linear);
-
             if (m_call_type == 0) {
                 std::stringstream declStream;
                 declStream << "declare " << m_type_str << " " << m_op_str
-                           << "(" << m_lhs->getTypeStr() << " , " << m_rhs->getTypeStr() << ")\n";
-
-                str_map_iter loc = declStrs.find(declStream.str());
-                if (loc == declStrs.end()) {
-                    declStrs[declStream.str()] = true;
-                }
+                           << "(" << m_children[0]->getTypeStr() << " , "
+                           << m_children[1]->getTypeStr() << ")\n";
+                declStrs[declStream.str()] = true;
 
-                kerStream << "%val" << m_id << " = call "
+                kerStream << "%val" << ids.id << " = call "
                           << m_type_str << " "
                           << m_op_str << "("
-                          << m_lhs->getTypeStr() << " "
-                          << "%val" << m_lhs->getId() << ", "
-                          << m_rhs->getTypeStr() << " "
-                          << "%val" << m_rhs->getId() << ")\n";
+                          << m_children[0]->getTypeStr() << " "
+                          << "%val" << ids.child_ids[0] << ", "
+                          << m_children[1]->getTypeStr() << " "
+                          << "%val" << ids.child_ids[1] << ")\n";
 
             } else {
                 if (m_call_type == 1) {
                     // arithmetic operations
-                    kerStream << "%val" << m_id << " = "
+                    kerStream << "%val" << ids.id << " = "
                               << m_op_str << " "
                               << m_type_str << " "
-                              << "%val" << m_lhs->getId() << ", "
-                              << "%val" << m_rhs->getId() << "\n";
+                              << "%val" << ids.child_ids[0] << ", "
+                              << "%val" << ids.child_ids[1] << "\n";
                 } else {
                     // logical operators
-                    kerStream << "%tmp" << m_id << " = "
+                    kerStream << "%tmp" << ids.id << " = "
                               << m_op_str << " "
-                              << m_lhs->getTypeStr() << " "
-                              << "%val" << m_lhs->getId() << ", "
-                              << "%val" << m_rhs->getId() << "\n";
+                              << m_children[0]->getTypeStr() << " "
+                              << "%val" << ids.child_ids[0] << ", "
+                              << "%val" << ids.child_ids[1] << "\n";
 
-                    kerStream << "%val" << m_id << " = "
-                              << "zext i1 %tmp" << m_id << " to i8\n";
+                    kerStream << "%val" << ids.id << " = "
+                              << "zext i1 %tmp" << ids.id << " to i8\n";
 
                 }
             }
-            m_gen_func = true;
-        }
-
-        int setId(int id)
-        {
-            if (m_set_id) return id;
-            id = m_lhs->setId(id);
-            id = m_rhs->setId(id);
-            m_id = id;
-            m_set_id = true;
-            return m_id + 1;
-        }
-
-        void getInfo(unsigned &len, unsigned &buf_count, unsigned &bytes)
-        {
-            if (m_set_id) return;
-            m_lhs->getInfo(len, buf_count, bytes);
-            m_rhs->getInfo(len, buf_count, bytes);
-            len++;
-            m_set_id = true;
-            return;
-        }
-
-        void resetFlags()
-        {
-            if (m_set_id) {
-                resetCommonFlags();
-                m_lhs->resetFlags();
-                m_rhs->resetFlags();
-            }
-        }
-
-        void setArgs(std::vector<void *> &args, bool is_linear)
-        {
-            if (m_set_arg) return;
-            m_lhs->setArgs(args, is_linear);
-            m_rhs->setArgs(args, is_linear);
-            m_set_arg = true;
         }
     };
 
diff --git a/src/backend/cuda/JIT/BufferNode.hpp b/src/backend/cuda/JIT/BufferNode.hpp
index bdc692b447..c6a03d2899 100644
--- a/src/backend/cuda/JIT/BufferNode.hpp
+++ b/src/backend/cuda/JIT/BufferNode.hpp
@@ -11,6 +11,7 @@
 #include "Node.hpp"
 #include <iomanip>
 #include <memory>
+#include <mutex>
 
 namespace cuda
 {
@@ -34,163 +35,132 @@ namespace JIT
         std::shared_ptr<T> m_data;
         Param<T> m_param;
         unsigned m_bytes;
+        std::once_flag m_set_data_flag;
 
         bool m_linear_buffer;
     public:
 
         BufferNode(const char *type_str,
                    const char *name_str)
-            : Node(type_str, name_str, 0)
+            : Node(type_str, name_str, 0, {})
         {
         }
 
         void setData(Param<T> param, std::shared_ptr<T> data, const unsigned bytes, bool is_linear)
         {
-            m_param = param;
-            m_data = data;
-            m_bytes = bytes;
-            m_linear_buffer = is_linear;
+            std::call_once(m_set_data_flag, [this, param, data, bytes, is_linear]() {
+                    m_param = param;
+                    m_data = data;
+                    m_bytes = bytes;
+                    m_linear_buffer = is_linear;
+                });
         }
 
         bool isLinear(dim_t dims[4])
         {
-            if (!m_set_is_linear) {
-                bool same_dims = true;
-                for (int i = 0; same_dims && i < 4; i++) {
-                    same_dims &= (dims[i] == m_param.dims[i]);
-                }
-                m_linear = m_linear_buffer && same_dims;
-                m_set_is_linear = true;
+            bool same_dims = true;
+            for (int i = 0; same_dims && i < 4; i++) {
+                same_dims &= (dims[i] == m_param.dims[i]);
             }
-            return m_linear;
+            return m_linear_buffer && same_dims;
         }
 
         bool isBuffer() { return true; }
 
-        void genKerName(std::stringstream &kerStream)
+        void genKerName(std::stringstream &kerStream, Node_ids ids)
         {
-            if (m_gen_name) return;
-
             kerStream << "_" << m_name_str;
-            kerStream << std::setw(2) << std::setfill('0') << std::hex << m_id << std::dec;
-            m_gen_name = true;
+            kerStream << std::setw(2) << std::setfill('0') << std::hex << ids.id << std::dec;
         }
 
         void genParams(std::stringstream &kerStream,
-                       std::stringstream &annStream, bool is_linear)
+                       std::stringstream &annStream,
+                       int id,
+                       bool is_linear)
         {
-            if (m_gen_param) return;
-            kerStream << m_type_str << "* %in" << m_id << ",\n";
+            kerStream << m_type_str << "* %in" << id << ",\n";
             annStream << m_type_str << "*,\n";
 
             if (!is_linear) {
-                kerStream << "i32 %dim0"     << m_id << ","
-                          << "i32 %dim1"     << m_id << ","
-                          << "i32 %dim2"     << m_id << ","
-                          << "i32 %dim3"     << m_id << ","
+                kerStream << "i32 %dim0"     << id << ","
+                          << "i32 %dim1"     << id << ","
+                          << "i32 %dim2"     << id << ","
+                          << "i32 %dim3"     << id << ","
                           << "\n"
-                          << "i32 %str1"     << m_id << ","
-                          << "i32 %str2"     << m_id << ","
-                          << "i32 %str3"     << m_id << ","
+                          << "i32 %str1"     << id << ","
+                          << "i32 %str2"     << id << ","
+                          << "i32 %str3"     << id << ","
                           << "\n";
 
                 annStream << "i32, i32, i32, i32,\n"
                           << "i32, i32, i32,\n";
             }
-
-            m_gen_param = true;
         }
 
-        void genOffsets(std::stringstream &kerStream, bool is_linear)
+        void genOffsets(std::stringstream &kerStream, int id, bool is_linear)
         {
-            if (m_gen_offset) return;
-
             if (!is_linear) {
-                kerStream << "%b3" << m_id << " = icmp slt i32 %id3, %dim3" << m_id << "\n";
-                kerStream << "%b2" << m_id << " = icmp slt i32 %id2, %dim2" << m_id << "\n";
-                kerStream << "%b1" << m_id << " = icmp slt i32 %id1, %dim1" << m_id << "\n";
-                kerStream << "%b0" << m_id << " = icmp slt i32 %id0, %dim0" << m_id << "\n";
+                kerStream << "%b3" << id << " = icmp slt i32 %id3, %dim3" << id << "\n";
+                kerStream << "%b2" << id << " = icmp slt i32 %id2, %dim2" << id << "\n";
+                kerStream << "%b1" << id << " = icmp slt i32 %id1, %dim1" << id << "\n";
+                kerStream << "%b0" << id << " = icmp slt i32 %id0, %dim0" << id << "\n";
 
-                kerStream << "%c3" << m_id << " = zext i1 %b3" << m_id << " to i32\n";
-                kerStream << "%c2" << m_id << " = zext i1 %b2" << m_id << " to i32\n";
-                kerStream << "%c1" << m_id << " = zext i1 %b1" << m_id << " to i32\n";
-                kerStream << "%c0" << m_id << " = zext i1 %b0" << m_id << " to i32\n";
+                kerStream << "%c3" << id << " = zext i1 %b3" << id << " to i32\n";
+                kerStream << "%c2" << id << " = zext i1 %b2" << id << " to i32\n";
+                kerStream << "%c1" << id << " = zext i1 %b1" << id << " to i32\n";
+                kerStream << "%c0" << id << " = zext i1 %b0" << id << " to i32\n";
 
-                kerStream << "%d3" << m_id << " = mul i32 %c3" << m_id << ", %id3\n";
-                kerStream << "%d2" << m_id << " = mul i32 %c2" << m_id << ", %id2\n";
-                kerStream << "%d1" << m_id << " = mul i32 %c1" << m_id << ", %id1\n";
-                kerStream << "%d0" << m_id << " = mul i32 %c0" << m_id << ", %id0\n";
+                kerStream << "%d3" << id << " = mul i32 %c3" << id << ", %id3\n";
+                kerStream << "%d2" << id << " = mul i32 %c2" << id << ", %id2\n";
+                kerStream << "%d1" << id << " = mul i32 %c1" << id << ", %id1\n";
+                kerStream << "%d0" << id << " = mul i32 %c0" << id << ", %id0\n";
 
-                kerStream << "%off3i" << m_id << " = mul i32 %d3" << m_id
-                          << ", %str3" << m_id << "\n";
+                kerStream << "%off3i" << id << " = mul i32 %d3" << id
+                          << ", %str3" << id << "\n";
 
-                kerStream << "%off2i" << m_id << " = mul i32 %d2" << m_id
-                          << ", %str2" << m_id << "\n";
+                kerStream << "%off2i" << id << " = mul i32 %d2" << id
+                          << ", %str2" << id << "\n";
 
-                kerStream << "%off1i" << m_id << " = mul i32 %d1" << m_id
-                          << ", %str1" << m_id << "\n";
+                kerStream << "%off1i" << id << " = mul i32 %d1" << id
+                          << ", %str1" << id << "\n";
 
-                kerStream << "%off23i" << m_id << " = add i32 %off2i"
-                          << m_id << ", %off3i" << m_id << "\n";
+                kerStream << "%off23i" << id << " = add i32 %off2i"
+                          << id << ", %off3i" << id << "\n";
 
-                kerStream << "%off123i" << m_id << " = add i32 %off23i"
-                          << m_id << ", %off1i" << m_id << "\n";
+                kerStream << "%off123i" << id << " = add i32 %off23i"
+                          << id << ", %off1i" << id << "\n";
 
-                kerStream << "%idxa" << m_id << " = add i32 %off123i"
-                          << m_id << ", %d0" << m_id << "\n";
+                kerStream << "%idxa" << id << " = add i32 %off123i"
+                          << id << ", %d0" << id << "\n";
 
-                kerStream << "%idx" << m_id << " = sext i32 %idxa" << m_id <<" to i64\n\n";
+                kerStream << "%idx" << id << " = sext i32 %idxa" << id <<" to i64\n\n";
             }
-
-            m_gen_offset = true;
         }
 
-        void genFuncs(std::stringstream &kerStream, str_map_t &declStrs, bool is_linear)
+        void genFuncs(std::stringstream &kerStream, str_map_t &declStrs, Node_ids ids, bool is_linear)
         {
-            if (m_gen_func) return;
-
-            kerStream << "%inIdx" << m_id << " = "
-                      << "getelementptr inbounds " << m_type_str << "* %in" << m_id
+            kerStream << "%inIdx" << ids.id << " = "
+                      << "getelementptr inbounds " << m_type_str << "* %in" << ids.id
                       << ", i64 %idx";
 
-            if (!is_linear) kerStream << m_id;
+            if (!is_linear) kerStream << ids.id;
             kerStream << "\n";
 
-            kerStream << "%val" << m_id << " = " << "load "
-                      << m_type_str << "* %inIdx" << m_id << "\n\n";
-
-            m_gen_func = true;
-        }
-
-        int setId(int id)
-        {
-            if (m_set_id) return id;
-
-            m_id = id;
-            m_set_id = true;
+            kerStream << "%val" << ids.id << " = " << "load "
+                      << m_type_str << "* %inIdx" << ids.id << "\n\n";
 
-            return m_id + 1;
         }
 
         void getInfo(unsigned &len, unsigned &buf_count, unsigned &bytes)
         {
-            if (m_set_id) return;
-
             len++;
             buf_count++;
             bytes += m_bytes;
-            m_set_id = true;
             return;
         }
 
-        void resetFlags()
-        {
-            if (m_set_id) resetCommonFlags();
-        }
-
         void setArgs(std::vector<void *> &args, bool is_linear)
         {
-            if (m_set_arg) return;
             args.push_back((void *)&(m_param.ptr));
 
             if (!is_linear) {
@@ -202,7 +172,6 @@ namespace JIT
                 args.push_back((void *)&m_param.strides[2]);
                 args.push_back((void *)&m_param.strides[3]);
             }
-            m_set_arg = true;
         }
     };
 
diff --git a/src/backend/cuda/JIT/Node.hpp b/src/backend/cuda/JIT/Node.hpp
index bd4307dd1e..035692550d 100644
--- a/src/backend/cuda/JIT/Node.hpp
+++ b/src/backend/cuda/JIT/Node.hpp
@@ -10,7 +10,7 @@
 #pragma once
 #include <optypes.hpp>
 
-#include <map>
+#include <unordered_map>
 #include <memory>
 #include <string>
 #include <vector>
@@ -20,97 +20,77 @@ namespace cuda
 
 namespace JIT
 {
-    typedef std::map<std::string, bool> str_map_t;
+    class Node;
+
+    typedef struct
+    {
+        int id;
+        std::vector<int> child_ids;
+    } Node_ids;
+
+    typedef std::unordered_map<std::string, bool> str_map_t;
     typedef str_map_t::iterator str_map_iter;
+    typedef std::shared_ptr<Node> Node_ptr;
+    typedef std::unordered_map<Node *, Node_ids> Node_map_t;
+    typedef Node_map_t::iterator Node_map_iter;
 
     class Node
     {
     protected:
         const std::string m_type_str;
         const std::string m_name_str;
-        int m_id;
         const int m_height;
-        bool m_set_id;
-        bool m_gen_func;
-        bool m_gen_param;
-        bool m_gen_offset;
-        bool m_set_arg;
-        bool m_gen_name;
-        bool m_linear;
-        bool m_set_is_linear;
-
-    protected:
-
-        void resetCommonFlags()
-        {
-            m_set_id = false;
-            m_gen_func = false;
-            m_gen_param = false;
-            m_gen_offset = false;
-            m_set_arg = false;
-            m_gen_name = false;
-            m_linear = false;
-            m_set_is_linear = false;
-        }
-
+        const std::vector<Node_ptr> m_children;
 
     public:
 
-        Node(const char *type_str, const char *name_str, const int height)
+        Node(const char *type_str, const char *name_str, const int height,
+             const std::vector<Node_ptr> children)
             : m_type_str(type_str),
               m_name_str(name_str),
-              m_id(-1),
               m_height(height),
-              m_set_id(false),
-              m_gen_func(false),
-              m_gen_param(false),
-              m_gen_offset(false),
-              m_set_arg(false),
-              m_gen_name(false),
-              m_linear(false),
-              m_set_is_linear(false)
+              m_children(children)
         {}
 
-        virtual void genKerName(std::stringstream &kerStream) {}
+        void getNodesMap(Node_map_t &node_map)
+        {
+            if (node_map.find(this) == node_map.end()) {
+                Node_ids ids;
+                for (const auto &child : m_children) {
+                    child->getNodesMap(node_map);
+                    ids.child_ids.push_back(node_map[child.get()].id);
+                }
+                ids.id = node_map.size();
+                node_map[this] = ids;
+            }
+        }
+
+        virtual void genKerName(std::stringstream &kerStream, Node_ids ids) {}
         virtual void genParams  (std::stringstream &kerStream,
-                                 std::stringstream &annStream, bool is_linear) {}
-        virtual void genOffsets (std::stringstream &kerStream, bool is_linear) {}
-        virtual void genFuncs   (std::stringstream &kerStream, str_map_t &declStrs, bool is_linear)
-        { m_gen_func = true;}
+                                 std::stringstream &annStream,
+                                 int id, bool is_linear) {}
+        virtual void genOffsets (std::stringstream &kerStream, int id, bool is_linear) {}
+        virtual void genFuncs   (std::stringstream &kerStream, str_map_t &declStrs,
+                                 Node_ids id, bool is_linear)
+        {}
 
-        virtual int setId(int id) { m_set_id = true; return id; }
-        virtual void setArgs(std::vector<void *> &args, bool is_linear) { m_set_arg = true; }
+        virtual void setArgs(std::vector<void *> &args, bool is_linear) {}
         virtual bool isLinear(dim_t dims[4]) { return true; }
 
-        virtual void resetFlags()
-        {
-            resetCommonFlags();
-        }
-
         virtual void getInfo(unsigned &len, unsigned &buf_count, unsigned &bytes)
         {
-            len = 0;
-            buf_count = 0;
-            bytes = 0;
+            len++;
         }
 
         virtual bool isBuffer() { return false; }
 
         std::string getTypeStr() { return m_type_str; }
 
-        bool isGenFunc() { return m_gen_func; }
-        bool isGenParam() { return m_gen_param; }
-        bool isGenOffset() { return m_gen_offset; }
-
-        int getId()  { return m_id; }
         int getHeight()  { return m_height; }
         std::string getNameStr() { return m_name_str; }
 
         virtual ~Node() {}
     };
-
-    typedef std::shared_ptr<Node> Node_ptr;
-
 }
 
 }
diff --git a/src/backend/cuda/JIT/ScalarNode.hpp b/src/backend/cuda/JIT/ScalarNode.hpp
index f7a1e33d99..264b7be61e 100644
--- a/src/backend/cuda/JIT/ScalarNode.hpp
+++ b/src/backend/cuda/JIT/ScalarNode.hpp
@@ -26,73 +26,29 @@ namespace JIT
     public:
 
         ScalarNode(T val)
-            : Node(irname<T>(), afShortName<T>(false), 0),
+            : Node(irname<T>(), afShortName<T>(false), 0, {}),
               m_val(val)
         {
         }
 
-        bool isLinear(dim_t dims[4])
+        void genKerName(std::stringstream &kerStream, Node_ids ids)
         {
-            return true;
-        }
-
-        void genKerName(std::stringstream &kerStream)
-        {
-            if (m_gen_name) return;
-
             kerStream << "_" << m_name_str;
-            kerStream << std::setw(2) << std::setfill('0') << std::hex << m_id << std::dec;
-            m_gen_name = true;
+            kerStream << std::setw(2) << std::setfill('0') << std::hex << ids.id << std::dec;
         }
 
         void genParams(std::stringstream &kerStream,
                        std::stringstream &annStream,
+                       int id,
                        bool is_linear)
         {
-            if (m_gen_param) return;
-            kerStream << m_type_str << " %val" << m_id << ", " << std::endl;
+            kerStream << m_type_str << " %val" << id << ", " << std::endl;
             annStream << m_type_str << ",\n";
-            m_gen_param = true;
-        }
-
-        void genOffsets(std::stringstream &kerStream, bool is_linear)
-        {
-            if (m_gen_offset) return;
-            m_gen_offset = true;
-        }
-
-        void genFuncs(std::stringstream &kerStream, str_map_t &declStrs, bool is_linear)
-        {
-            if (m_gen_func) return;
-            m_gen_func = true;
-        }
-
-        int setId(int id)
-        {
-            if (m_set_id) return id;
-            m_id = id;
-            m_set_id = true;
-            return m_id + 1;
-        }
-
-        void getInfo(unsigned &len, unsigned &buf_count, unsigned &bytes)
-        {
-            if (m_set_id) return;
-            len++;
-            m_set_id = true;
-            return;
-        }
-
-        void resetFlags()
-        {
-            if (m_set_id) resetCommonFlags();
         }
 
         void setArgs(std::vector<void *> &args, bool is_linear)
         {
-            if (m_set_arg) return;
             args.push_back((void *)&m_val);
-            m_set_arg = true;
         }
     };
 }
diff --git a/src/backend/cuda/JIT/UnaryNode.hpp b/src/backend/cuda/JIT/UnaryNode.hpp
index a85e05371c..ac334c14b8 100644
--- a/src/backend/cuda/JIT/UnaryNode.hpp
+++ b/src/backend/cuda/JIT/UnaryNode.hpp
@@ -20,139 +20,64 @@ namespace JIT
     class UnaryNode : public Node
     {
     private:
-        std::string m_op_str;
-        Node_ptr m_child;
-        int m_op;
-        bool m_is_check;
+        const std::string m_op_str;
+        const int m_op;
+        const bool m_is_check;
 
     public:
         UnaryNode(const char *out_type_str, const char *name_str,
                   const std::string &op_str,
                   Node_ptr child, int op, bool is_check=false)
-            : Node(out_type_str, name_str, child->getHeight() + 1),
+            : Node(out_type_str, name_str, child->getHeight() + 1, {child}),
               m_op_str(op_str),
-              m_child(child),
               m_op(op),
               m_is_check(is_check)
         {
         }
 
-        bool isLinear(dim_t dims[4])
+        void genKerName(std::stringstream &kerStream, Node_ids ids)
         {
-            if (!m_set_is_linear) {
-                m_linear = m_child->isLinear(dims);
-                m_set_is_linear = true;
-            }
-            return m_linear;
-        }
-
-        void genParams(std::stringstream &kerStream,
-                       std::stringstream &annStream, bool is_linear)
-        {
-            if (m_gen_param) return;
-            if (!(m_child->isGenParam())) m_child->genParams(kerStream, annStream, is_linear);
-            m_gen_param = true;
-        }
-
-
-        void genOffsets(std::stringstream &kerStream, bool is_linear)
-        {
-            if (m_gen_offset) return;
-            if (!(m_child->isGenOffset())) m_child->genOffsets(kerStream, is_linear);
-            m_gen_offset = true;
-        }
-
-        void genKerName(std::stringstream &kerStream)
-        {
-            if (m_gen_name) return;
-
-            m_child->genKerName(kerStream);
-
             // Make the hex representation of enum part of the Kernel name
             kerStream << "_" << std::setw(2) << std::setfill('0') << std::hex << m_op;
-            kerStream << std::setw(2) << std::setfill('0') << std::hex << m_child->getId();
-            kerStream << std::setw(2) << std::setfill('0') << std::hex << m_id << std::dec;
-            m_gen_name = true;
+            kerStream << std::setw(2) << std::setfill('0') << std::hex << ids.child_ids[0];
+            kerStream << std::setw(2) << std::setfill('0') << std::hex << ids.id << std::dec;
         }
 
-        void genFuncs(std::stringstream &kerStream, str_map_t &declStrs, bool is_linear)
+        void genFuncs(std::stringstream &kerStream, str_map_t &declStrs, Node_ids ids, bool is_linear)
         {
-            if (m_gen_func) return;
-
-            if (!(m_child->isGenFunc())) m_child->genFuncs(kerStream, declStrs, is_linear);
-
             std::stringstream declStream;
 
             if (m_is_check) {
                 declStream << "declare " << "i32 " << m_op_str
-                           << "(" << m_child->getTypeStr() << ")\n";
+                           << "(" << m_children[0]->getTypeStr() << ")\n";
             } else {
                 declStream << "declare " << m_type_str << " " << m_op_str
-                           << "(" << m_child->getTypeStr() << ")\n";
+                           << "(" << m_children[0]->getTypeStr() << ")\n";
             }
 
-            str_map_iter loc = declStrs.find(declStream.str());
-            if (loc == declStrs.end()) {
-                declStrs[declStream.str()] = true;
-            }
+            declStrs[declStream.str()] = true;
 
             if (m_is_check) {
-                kerStream << "%tmp" << m_id << " = call i32 "
+                kerStream << "%tmp" << ids.id << " = call i32 "
                           << m_op_str << "("
-                          << m_child->getTypeStr() << " "
-                          << "%val" << m_child->getId() << ")\n";
+                          << m_children[0]->getTypeStr() << " "
+                          << "%val" << ids.child_ids[0] << ")\n";
 
                 if (m_type_str[0] == 'i') {
-                    kerStream << "%val" << m_id << " = "
-                              << "trunc i32 %tmp" << m_id << " to " << m_type_str << "\n";
+                    kerStream << "%val" << ids.id << " = "
+                              << "trunc i32 %tmp" << ids.id << " to " << m_type_str << "\n";
                 } else {
-                    kerStream << "%val" << m_id << " = "
-                              << "sitofp i32 %tmp" << m_id << " to " << m_type_str << "\n";
+                    kerStream << "%val" << ids.id << " = "
+                              << "sitofp i32 %tmp" << ids.id << " to " << m_type_str << "\n";
                 }
 
             } else {
-                kerStream << "%val" << m_id << " = call "
+                kerStream << "%val" << ids.id << " = call "
                           << m_type_str << " "
                           << m_op_str << "("
-                          << m_child->getTypeStr() << " "
-                          << "%val" << m_child->getId() << ")\n";
+                          << m_children[0]->getTypeStr() << " "
+                          << "%val" << ids.child_ids[0] << ")\n";
             }
-
-            m_gen_func = true;
-        }
-
-        int setId(int id)
-        {
-            if (m_set_id) return id;
-            id = m_child->setId(id);
-            m_id = id;
-            m_set_id = true;
-            return m_id + 1;
-        }
-
-        void getInfo(unsigned &len, unsigned &buf_count, unsigned &bytes)
-        {
-            if (m_set_id) return;
-            m_child->getInfo(len, buf_count, bytes);
-            len++;
-            m_set_id = true;
-            return;
-        }
-
-        void resetFlags()
-        {
-            if (m_set_id) {
-                m_is_check = false;
-                resetCommonFlags();
-                m_child->resetFlags();
-            }
-        }
-
-        void setArgs(std::vector<void *> &args, bool is_linear)
-        {
-            if (m_set_arg) return;
-            m_child->setArgs(args, is_linear);
-            m_set_arg = true;
         }
     };
 
diff --git a/src/backend/cuda/Param.hpp b/src/backend/cuda/Param.hpp
index c07aaa72b5..6bee8a5106 100644
--- a/src/backend/cuda/Param.hpp
+++ b/src/backend/cuda/Param.hpp
@@ -15,11 +15,25 @@ namespace cuda
 {
 
 template<typename T>
-struct Param
+class Param
 {
+public:
     T *ptr;
     dim_t dims[4];
     dim_t strides[4];
+
+    __DH__ Param() : ptr(nullptr)
+    {
+    }
+
+    __DH__ Param(T *iptr, const dim_t *idims, const dim_t *istrides) :
+        ptr(iptr)
+    {
+        for (int i = 0; i < 4; i++) {
+            dims[i] = idims[i];
+            strides[i] = istrides[i];
+        }
+    }
 };
 
 template<typename T>
diff --git a/src/backend/cuda/jit.cpp b/src/backend/cuda/jit.cpp
index f5e64551df..776c5c8f28 100644
--- a/src/backend/cuda/jit.cpp
+++ b/src/backend/cuda/jit.cpp
@@ -54,6 +54,8 @@ namespace cuda
 using JIT::Node;
 using JIT::str_map_iter;
 using JIT::str_map_t;
+using JIT::Node_ids;
+using JIT::Node_map_t;
 using std::hash;
 using std::map;
 using std::string;
@@ -67,7 +69,10 @@ const char *layout32 = "target datalayout = \"e-p:32:32:32-i1:8:8-i8:8:8-i16:16:
 const char *triple64 = "target triple = \"nvptx64-unknown-cuda\"\n\n";
 const char *triple32 = "target triple = \"nvptx-unknown-cuda\"\n\n";
 
-static string getFuncName(vector<Node *> nodes, bool is_linear)
+static string getFuncName(const vector<Node *> &output_nodes,
+                          const vector<Node *> &full_nodes,
+                          const vector<Node_ids> &full_ids,
+                          bool is_linear)
 {
     stringstream funcName;
     stringstream hashName;
@@ -75,14 +80,12 @@ static string getFuncName(vector<Node *> nodes, bool is_linear)
     if (is_linear) funcName << "L_"; //Kernel Linear
     else           funcName << "G_"; //Kernel General
 
-    int id = 0;
+    for (const auto &node : output_nodes) {
+        funcName << node->getNameStr() << "_";
+    }
 
-    for (int i = 0; i < (int)nodes.size(); i++) {
-        funcName << "[";
-        id = nodes[i]->setId(id);
-        funcName << nodes[i]->getNameStr();
-        nodes[i]->genKerName(funcName);
-        funcName << "]";
+    for (int i = 0; i < (int)full_nodes.size(); i++) {
+        full_nodes[i]->genKerName(funcName, full_ids[i]);
     }
 
     hash<string> hash_fn;
@@ -92,7 +95,11 @@ static string getFuncName(vector<Node *> nodes, bool is_linear)
     return hashName.str();
 }
 
-static string getKernelString(string funcName, vector<Node *> nodes, bool is_linear)
+static string getKernelString(const string funcName,
+                              const vector<Node *> &full_nodes,
+                              const vector<Node_ids> &full_ids,
+                              const vector<int> &output_ids,
+                              bool is_linear)
 {
     static const char *defineVoid = "define void ";
     static const char *generalDimParams = "\n"
@@ -218,15 +225,30 @@ static string getKernelString(string funcName, vector<Node *> nodes, bool is_lin
     stringstream outWriteStream;
     str_map_t declStrs;
 
-    for (int i = 0; i < (int)nodes.size(); i++) {
-        string outTypeStr = nodes[i]->getTypeStr();
-        int id = nodes[i]->getId();
+    vector<string> types_output(output_ids.size());
+    for (int i = 0; i < (int)output_ids.size(); i++) {
+        types_output[i] = full_nodes[output_ids[i]]->getTypeStr();
+    }
+
+    for (int i = 0; i < (int)full_nodes.size(); i++) {
+        const auto &node = full_nodes[i];
+        const auto &ids_curr = full_ids[i];
+        // Generate input parameters, needs only current id
+        node->genParams(inParamStream, inAnnStream, ids_curr.id, is_linear);
+        // Generate input offsets, needs only current id
+        node->genOffsets(offsetsStream, ids_curr.id, is_linear);
+        // Generate the core function body, needs children id as well
+        node->genFuncs(funcBodyStream, declStrs, ids_curr, is_linear);
+    }
 
-        nodes[i]->genParams(inParamStream, inAnnStream, is_linear);
+    for (int i = 0; i < (int)output_ids.size(); i++) {
+        int id = output_ids[i];
+        string outTypeStr = types_output[i];
+
+        // Generate output parameters
         outParamStream << outTypeStr << "* %out" << id << ",\n";
-        nodes[i]->genOffsets(offsetsStream, is_linear);
-        nodes[i]->genFuncs(funcBodyStream, declStrs, is_linear);
 
+        // Generate instruction to write output
         outWriteStream << "%outIdx" << id
                        << "= getelementptr inbounds "
                        << outTypeStr
@@ -237,6 +259,8 @@ static string getKernelString(string funcName, vector<Node *> nodes, bool is_lin
                        << " %val" << id << ", "
                        << outTypeStr
                        << "* %outIdx" << id << "\n";
+
+        // Generate output annotation string
         outAnnStream << outTypeStr << "*,\n";
     }
 
@@ -267,7 +291,8 @@ static string getKernelString(string funcName, vector<Node *> nodes, bool is_lin
               << outWriteStream.str()
               << blockEnd;
 
-    for(str_map_iter iterator = declStrs.begin(); iterator != declStrs.end(); iterator++) {
+    for(str_map_iter iterator = declStrs.begin();
+        iterator != declStrs.end(); iterator++) {
         kerStream << iterator->first << "\n";
     }
     kerStream << functionLoad;
@@ -528,20 +553,24 @@ static kc_entry_t compileKernel(const char *ker_name, string jit_ker)
     return entry;
 }
 
-static CUfunction getKernel(vector<Node *> nodes, bool is_linear)
+static CUfunction getKernel(const vector<Node *> &output_nodes,
+                            const vector<int> &output_ids,
+                            const vector<Node *> &full_nodes,
+                            const vector<Node_ids> &full_ids,
+                            const bool is_linear)
 {
     typedef std::map<std::string, kc_entry_t> kc_t;
 
     thread_local static kc_t kernelCaches[DeviceManager::MAX_DEVICES];
 
-    string funcName = getFuncName(nodes, is_linear);
+    string funcName = getFuncName(output_nodes, full_nodes, full_ids, is_linear);
     int device      = getActiveDeviceId();
 
     kc_t::iterator idx = kernelCaches[device].find(funcName);
     kc_entry_t entry = {NULL, NULL};
 
     if (idx == kernelCaches[device].end()) {
-        string jit_ker = getKernelString(funcName, nodes, is_linear);
+        string jit_ker = getKernelString(funcName, full_nodes, full_ids, output_ids, is_linear);
         entry = compileKernel(funcName.c_str(), jit_ker);
         kernelCaches[device][funcName] = entry;
     } else {
@@ -552,19 +581,31 @@ static CUfunction getKernel(vector<Node *> nodes, bool is_linear)
 }
 
 template<typename T>
-void evalNodes(vector<Param<T> >&outputs, vector<Node *> nodes)
+void evalNodes(vector<Param<T> >&outputs, vector<Node *> output_nodes)
 {
     int num_outputs = (int)outputs.size();
 
     if (num_outputs == 0) return;
 
-    bool is_linear = true;
+    Node_map_t nodes;
+    vector<int> output_ids;
+    for (auto &node : output_nodes) {
+        node->getNodesMap(nodes);
+        output_ids.push_back(nodes[node].id);
+    }
 
-    for (int i = 0; i < num_outputs; i++) {
-        is_linear &= nodes[i]->isLinear(outputs[0].dims);
+    vector<Node *> full_nodes(nodes.size());
+    vector<Node_ids> full_ids(nodes.size());
+    bool is_linear = true;
+    for (auto &map_entry : nodes) {
+        full_nodes[map_entry.second.id] = map_entry.first;
+        full_ids[map_entry.second.id] = map_entry.second;
+        is_linear &= map_entry.first->isLinear(outputs[0].dims);
     }
 
-    CUfunction ker = getKernel(nodes, is_linear);
+    CUfunction ker = getKernel(output_nodes, output_ids,
+                               full_nodes, full_ids,
+                               is_linear);
 
     int threads_x = 1, threads_y = 1;
     int blocks_x_ = 1, blocks_y_ = 1;
@@ -607,8 +648,8 @@ void evalNodes(vector<Param<T> >&outputs, vector<Node *> nodes)
 
     vector<void *> args;
 
-    for (int i = 0; i < num_outputs; i++) {
-        nodes[i]->setArgs(args, is_linear);
+    for (const auto &node : full_nodes) {
+        node->setArgs(args, is_linear);
     }
 
     for (int i = 0; i < num_outputs; i++) {
@@ -658,21 +699,17 @@ void evalNodes(vector<Param<T> >&outputs, vector<Node *> nodes)
                             getActiveStream(),
                             &args.front(),
                             NULL));
-
-    for (int i = 0; i < num_outputs; i++) {
-        nodes[i]->resetFlags();
-    }
 }
 
 template<typename T>
 void evalNodes(Param<T> &out, Node *node)
 {
     vector<Param<T>> outputs;
-    vector<Node *> nodes;
+    vector<Node *> output_nodes;
 
     outputs.push_back(out);
-    nodes.push_back(node);
-    evalNodes(outputs, nodes);
+    output_nodes.push_back(node);
+    evalNodes(outputs, output_nodes);
     return;
 }
 

From 9ffd295aadb2f02960bb679068c3418c2ca2f44e Mon Sep 17 00:00:00 2001
From: Pavan Yalamanchili <contact@pavanky.com>
Date: Tue, 14 Mar 2017 23:43:45 -0700
Subject: [PATCH 49/82] Changes required to make JIT in OpenCL backend thread
 safe

---
 src/backend/opencl/Array.cpp          |  27 ++++++-
 src/backend/opencl/Array.hpp          |   6 +-
 src/backend/opencl/JIT/BinaryNode.hpp | 102 +++---------------------
 src/backend/opencl/JIT/BufferNode.hpp |  90 +++++++--------------
 src/backend/opencl/JIT/Node.hpp       |  93 +++++++++-------------
 src/backend/opencl/JIT/ScalarNode.hpp |  59 ++------------
 src/backend/opencl/JIT/UnaryNode.hpp  |  88 +++------------------
 src/backend/opencl/jit.cpp            | 108 ++++++++++++++++----------
 8 files changed, 179 insertions(+), 394 deletions(-)

diff --git a/src/backend/opencl/Array.cpp b/src/backend/opencl/Array.cpp
index 88355ee3dd..1ab28a8e31 100644
--- a/src/backend/opencl/Array.cpp
+++ b/src/backend/opencl/Array.cpp
@@ -242,8 +242,13 @@ namespace opencl
 
                     unsigned length =0, buf_count = 0, bytes = 0;
                     Node *n = node.get();
-                    n->getInfo(length, buf_count, bytes);
-                    n->resetFlags();
+                    JIT::Node_map_t nodes_map;
+                    n->getNodesMap(nodes_map);
+
+                    for(auto &entry : nodes_map) {
+                        Node *node = entry.first;
+                        node->getInfo(length, buf_count, bytes);
+                    }
 
                     if (2 * bytes > lock_bytes) {
                         out.eval();
@@ -379,6 +384,17 @@ namespace opencl
         return;
     }
 
+    template<typename T>
+    void
+    Array<T>::setDataDims(const dim4 &new_dims)
+    {
+        modDims(new_dims);
+        data_dims = new_dims;
+        if (node->isBuffer()) {
+            node = bufferNodePtr<T>();
+        }
+    }
+
 #define INSTANTIATE(T)                                                  \
     template       Array<T>  createHostDataArray<T>   (const dim4 &size, const T * const data); \
     template       Array<T>  createDeviceDataArray<T> (const dim4 &size, const void *data); \
@@ -400,9 +416,12 @@ namespace opencl
     template       void Array<T>::eval();                               \
     template       void Array<T>::eval() const;                         \
     template       cl::Buffer* Array<T>::device();                      \
-    template       void      writeHostDataArray<T>    (Array<T> &arr, const T * const data, const size_t bytes); \
-    template       void      writeDeviceDataArray<T>  (Array<T> &arr, const void * const data, const size_t bytes); \
+    template       void      writeHostDataArray<T>    (Array<T> &arr, const T * const data, \
+                                                       const size_t bytes); \
+    template       void      writeDeviceDataArray<T>  (Array<T> &arr, const void * const data, \
+                                                       const size_t bytes); \
     template       void      evalMultiple<T>     (std::vector<Array<T>*> arrays); \
+    template       void Array<T>::setDataDims(const dim4 &new_dims);    \
 
     INSTANTIATE(float)
     INSTANTIATE(double)
diff --git a/src/backend/opencl/Array.hpp b/src/backend/opencl/Array.hpp
index 64ac5c742f..f7c36b697b 100644
--- a/src/backend/opencl/Array.hpp
+++ b/src/backend/opencl/Array.hpp
@@ -204,11 +204,7 @@ namespace opencl
             return data_dims;
         }
 
-        void setDataDims(const dim4 &new_dims)
-        {
-            modDims(new_dims);
-            data_dims = new_dims;
-        }
+        void setDataDims(const dim4 &new_dims);
 
         size_t getAllocatedBytes() const
         {
diff --git a/src/backend/opencl/JIT/BinaryNode.hpp b/src/backend/opencl/JIT/BinaryNode.hpp
index 5aa98810bd..c2f605f6e2 100644
--- a/src/backend/opencl/JIT/BinaryNode.hpp
+++ b/src/backend/opencl/JIT/BinaryNode.hpp
@@ -21,117 +21,33 @@ namespace JIT
     {
     private:
         std::string m_op_str;
-        Node_ptr m_lhs, m_rhs;
         int m_op;
 
     public:
         BinaryNode(const char *out_type_str, const char *name_str,
                    const char *op_str,
                    Node_ptr lhs, Node_ptr rhs, int op)
-            : Node(out_type_str, name_str, std::max(lhs->getHeight(), rhs->getHeight()) + 1),
+            : Node(out_type_str, name_str, std::max(lhs->getHeight(), rhs->getHeight()) + 1, {lhs, rhs}),
               m_op_str(op_str),
-              m_lhs(lhs),
-              m_rhs(rhs),
               m_op(op)
         {
         }
 
-        bool isLinear(dim_t dims[4])
+        void genKerName(std::stringstream &kerStream, Node_ids ids)
         {
-            if (!m_set_is_linear) {
-                m_linear = m_lhs->isLinear(dims) && m_rhs->isLinear(dims);
-                m_set_is_linear = true;
-            }
-            return m_linear;
-        }
-
-        void genParams(std::stringstream &kerStream, bool is_linear)
-        {
-            if (m_gen_param) return;
-            if (!(m_lhs->isGenParam())) m_lhs->genParams(kerStream, is_linear);
-            if (!(m_rhs->isGenParam())) m_rhs->genParams(kerStream, is_linear);
-            m_gen_param = true;
-        }
-
-        int setArgs(cl::Kernel &ker, int id, bool is_linear)
-        {
-            if (m_set_arg) return id;
-            m_set_arg = true;
-
-            id = m_lhs->setArgs(ker, id, is_linear);
-            id = m_rhs->setArgs(ker, id, is_linear);
-            return id;
-        }
-
-        void genOffsets(std::stringstream &kerStream, bool is_linear)
-        {
-            if (m_gen_offset) return;
-            if (!(m_lhs->isGenOffset())) m_lhs->genOffsets(kerStream, is_linear);
-            if (!(m_rhs->isGenOffset())) m_rhs->genOffsets(kerStream, is_linear);
-            m_gen_offset = true;
-        }
-
-        void genKerName(std::stringstream &kerStream)
-        {
-            if (m_gen_name) return;
-            m_lhs->genKerName(kerStream);
-            m_rhs->genKerName(kerStream);
-
             // Make the dec representation of enum part of the Kernel name
             kerStream << "_" << std::setw(3) << std::setfill('0') << std::dec << m_op;
-            kerStream << std::setw(3) << std::setfill('0') << std::dec << m_lhs->getId();
-            kerStream << std::setw(3) << std::setfill('0') << std::dec << m_rhs->getId();
-            kerStream << std::setw(3) << std::setfill('0') << std::dec << m_id << std::dec;
-            m_gen_name = true;
+            kerStream << std::setw(3) << std::setfill('0') << std::dec << ids.child_ids[0];
+            kerStream << std::setw(3) << std::setfill('0') << std::dec << ids.child_ids[1];
+            kerStream << std::setw(3) << std::setfill('0') << std::dec << ids.id << std::dec;
         }
 
-        void genFuncs(std::stringstream &kerStream)
+        void genFuncs(std::stringstream &kerStream, Node_ids ids)
         {
-            if (m_gen_func) return;
-
-            if (!(m_lhs->isGenFunc())) m_lhs->genFuncs(kerStream);
-            if (!(m_rhs->isGenFunc())) m_rhs->genFuncs(kerStream);
-
-            kerStream << m_type_str << " val" << m_id << " = "
-                      << m_op_str << "(val" << m_lhs->getId()
-                      << ", val" << m_rhs->getId() << ");"
+            kerStream << m_type_str << " val" << ids.id << " = "
+                      << m_op_str << "(val" << ids.child_ids[0]
+                      << ", val" << ids.child_ids[1] << ");"
                       << "\n";
-
-            m_gen_func = true;
-        }
-
-        int setId(int id)
-        {
-            if (m_set_id) return id;
-
-            id = m_lhs->setId(id);
-            id = m_rhs->setId(id);
-
-            m_id = id;
-            m_set_id = true;
-
-            return m_id + 1;
-        }
-
-        void getInfo(unsigned &len, unsigned &buf_count, unsigned &bytes)
-        {
-            if (m_set_id) return;
-
-            m_lhs->getInfo(len, buf_count, bytes);
-            m_rhs->getInfo(len, buf_count, bytes);
-            len++;
-
-            m_set_id = true;
-            return;
-        }
-
-        void resetFlags()
-        {
-            if (m_set_id) {
-                resetCommonFlags();
-                m_lhs->resetFlags();
-                m_rhs->resetFlags();
-            }
         }
     };
 
diff --git a/src/backend/opencl/JIT/BufferNode.hpp b/src/backend/opencl/JIT/BufferNode.hpp
index 280c796a79..92a2c32697 100644
--- a/src/backend/opencl/JIT/BufferNode.hpp
+++ b/src/backend/opencl/JIT/BufferNode.hpp
@@ -10,6 +10,7 @@
 #pragma once
 #include "Node.hpp"
 #include <iomanip>
+#include <mutex>
 
 namespace opencl
 {
@@ -25,12 +26,13 @@ namespace JIT
         KParam m_info;
         unsigned m_bytes;
         bool m_linear_buffer;
+        std::once_flag m_set_data_flag;
 
     public:
 
         BufferNode(const char *type_str,
                    const char *name_str)
-            : Node(type_str, name_str, 0)
+            : Node(type_str, name_str, 0, {})
         {
         }
 
@@ -42,70 +44,55 @@ namespace JIT
 
         void setData(KParam info, std::shared_ptr<cl::Buffer> data, const unsigned bytes, bool is_linear)
         {
-            m_info = info;
-            m_data = data;
-            m_bytes = bytes;
-            m_linear_buffer = is_linear;
+            std::call_once(m_set_data_flag, [this, info, data, bytes, is_linear]() {
+                    m_info = info;
+                    m_data = data;
+                    m_bytes = bytes;
+                    m_linear_buffer = is_linear;
+                });
         }
 
         bool isLinear(dim_t dims[4])
         {
-            if (!m_set_is_linear) {
-                bool same_dims = true;
-                for (int i = 0; same_dims && i < 4; i++) {
-                    same_dims &= (dims[i] == m_info.dims[i]);
-                }
-                m_set_is_linear = true;
-                m_linear = m_linear_buffer && same_dims;
+            bool same_dims = true;
+            for (int i = 0; same_dims && i < 4; i++) {
+                same_dims &= (dims[i] == m_info.dims[i]);
             }
-            return m_linear;
+            return m_linear_buffer && same_dims;
         }
 
-        void genKerName(std::stringstream &kerStream)
+        void genKerName(std::stringstream &kerStream, Node_ids ids)
         {
-            if (m_gen_name) return;
-
             kerStream << "_" << m_name_str;
-            kerStream << std::setw(3) << std::setfill('0') << std::dec << m_id << std::dec;
-            m_gen_name = true;
+            kerStream << std::setw(3) << std::setfill('0') << std::dec << ids.id << std::dec;
         }
 
-        void genParams(std::stringstream &kerStream, bool is_linear)
+        void genParams(std::stringstream &kerStream, int id, bool is_linear)
         {
-            if (m_gen_param) return;
-
             if (!is_linear) {
-                kerStream << "__global " << m_type_str << " *in" << m_id
-                          << ", KParam iInfo" << m_id << ", " << "\n";
+                kerStream << "__global " << m_type_str << " *in" << id
+                          << ", KParam iInfo" << id << ", " << "\n";
             } else {
-                kerStream << "__global " << m_type_str << " *in" << m_id
-                          << ", dim_t iInfo" << m_id << "_offset, " << "\n";
+                kerStream << "__global " << m_type_str << " *in" << id
+                          << ", dim_t iInfo" << id << "_offset, " << "\n";
             }
-            m_gen_param = true;
         }
 
         int setArgs(cl::Kernel &ker, int id, bool is_linear)
         {
-            if (m_set_arg) return id;
-
             ker.setArg(id + 0, *m_data);
-
             if (!is_linear) {
                 ker.setArg(id + 1,  m_info);
             } else {
                 ker.setArg(id + 1, m_info.offset);
             }
-
-            m_set_arg = true;
             return id + 2;
         }
 
-        void genOffsets(std::stringstream &kerStream, bool is_linear)
+        void genOffsets(std::stringstream &kerStream, int id, bool is_linear)
         {
-            if (m_gen_offset) return;
-
-            std::string idx_str = std::string("int idx") + std::to_string(m_id);
-            std::string info_str = std::string("iInfo") + std::to_string(m_id);;
+            std::string idx_str = std::string("int idx") + std::to_string(id);
+            std::string info_str = std::string("iInfo") + std::to_string(id);;
 
             if (!is_linear) {
                 kerStream << idx_str << " = "
@@ -121,47 +108,22 @@ namespace JIT
             } else {
                 kerStream << idx_str << " = idx + " << info_str << "_offset;" << "\n";
             }
-
-            m_gen_offset = true;
         }
 
-        void genFuncs(std::stringstream &kerStream)
+        void genFuncs(std::stringstream &kerStream, Node_ids ids)
         {
-            if (m_gen_func) return;
-
-            kerStream << m_type_str << " val" << m_id << " = "
-                      << "in" << m_id << "[idx" << m_id << "];"
+            kerStream << m_type_str << " val" << ids.id << " = "
+                      << "in" << ids.id << "[idx" << ids.id << "];"
                       << "\n";
-
-            m_gen_func = true;
-        }
-
-        int setId(int id)
-        {
-            if (m_set_id) return id;
-
-            m_id = id;
-            m_set_id = true;
-
-            return m_id + 1;
         }
 
         void getInfo(unsigned &len, unsigned &buf_count, unsigned &bytes)
         {
-            if (m_set_id) return;
-
             len++;
             buf_count++;
             bytes += m_bytes;
-            m_set_id = true;
             return;
         }
-
-
-        void resetFlags()
-        {
-            if (m_set_id) resetCommonFlags();
-        }
     };
 
 }
diff --git a/src/backend/opencl/JIT/Node.hpp b/src/backend/opencl/JIT/Node.hpp
index abf7a2d908..c4e0176664 100644
--- a/src/backend/opencl/JIT/Node.hpp
+++ b/src/backend/opencl/JIT/Node.hpp
@@ -13,101 +13,78 @@
 #include <string>
 #include <vector>
 #include <memory>
+#include <unordered_map>
 
 namespace opencl
 {
 
 namespace JIT
 {
+
+    class Node;
     using std::shared_ptr;
+    typedef shared_ptr<Node> Node_ptr;
+
+    typedef struct
+    {
+        int id;
+        std::vector<int> child_ids;
+    } Node_ids;
+
+    typedef std::unordered_map<Node *, Node_ids> Node_map_t;
+    typedef Node_map_t::iterator Node_map_iter;
 
     class Node
     {
     protected:
         const std::string m_type_str;
         const std::string m_name_str;
-        int m_id;
         const int m_height;
-        bool m_set_id;
-        bool m_gen_func;
-        bool m_gen_param;
-        bool m_gen_offset;
-        bool m_set_arg;
-        bool m_gen_name;
-        bool m_linear;
-        bool m_set_is_linear;
-
-    protected:
-        void resetCommonFlags()
-        {
-            m_set_id = false;
-            m_gen_func = false;
-            m_gen_param = false;
-            m_gen_offset = false;
-            m_set_arg = false;
-            m_gen_name = false;
-            m_linear = false;
-            m_set_is_linear = false;
-        }
+        const std::vector<Node_ptr> m_children;
 
     public:
 
-        Node(const char *type_str, const char *name_str, const int height)
+        Node(const char *type_str, const char *name_str, const int height,
+             const std::vector<Node_ptr> children)
             : m_type_str(type_str),
               m_name_str(name_str),
-              m_id(-1),
               m_height(height),
-              m_set_id(false),
-              m_gen_func(false),
-              m_gen_param(false),
-              m_gen_offset(false),
-              m_set_arg(false),
-              m_gen_name(false),
-              m_linear(false),
-              m_set_is_linear(false)
+              m_children(children)
         {}
 
-        virtual void genKerName(std::stringstream &kerStream) {}
-        virtual void genParams  (std::stringstream &kerStream, bool is_linear) {}
-        virtual void genOffsets (std::stringstream &kerStream, bool is_linear) {}
-        virtual void genFuncs   (std::stringstream &kerStream) { m_gen_func = true;}
-
-        virtual int setArgs (cl::Kernel &ker, int id, bool is_linear) { return id; }
-
-        virtual int setId(int id) { m_set_id = true; return id; }
-
-        virtual void getInfo(unsigned &len, unsigned &buf_count, unsigned &bytes)
+        void getNodesMap(Node_map_t &node_map)
         {
-            len = 0;
-            buf_count = 0;
-            bytes = 0;
+            if (node_map.find(this) == node_map.end()) {
+                Node_ids ids;
+                for (const auto &child : m_children) {
+                    child->getNodesMap(node_map);
+                    ids.child_ids.push_back(node_map[child.get()].id);
+                }
+                ids.id = node_map.size();
+                node_map[this] = ids;
+            }
         }
 
-        virtual bool isBuffer() { return false; }
+        virtual void genKerName(std::stringstream &kerStream, Node_ids ids) {}
+        virtual void genParams  (std::stringstream &kerStream, int id, bool is_linear) {}
+        virtual void genOffsets (std::stringstream &kerStream, int id, bool is_linear) {}
+        virtual void genFuncs   (std::stringstream &kerStream, Node_ids) {}
 
+        virtual int setArgs (cl::Kernel &ker, int id, bool is_linear) { return id; }
 
-        virtual void resetFlags()
+        virtual void getInfo(unsigned &len, unsigned &buf_count, unsigned &bytes)
         {
-            resetCommonFlags();
+            len++;
         }
 
+        virtual bool isBuffer() { return false; }
         virtual bool isLinear(dim_t dims[4]) { return true; }
-
         std::string getTypeStr() { return m_type_str; }
-
-        bool isGenFunc() { return m_gen_func; }
-        bool isGenParam() { return m_gen_param; }
-        bool isGenOffset() { return m_gen_offset; }
-
-        int getId()  { return m_id; }
         int getHeight()  { return m_height; }
         std::string getNameStr() { return m_name_str; }
 
         virtual ~Node() {}
     };
-
-    typedef shared_ptr<Node> Node_ptr;
-
 }
 
 }
diff --git a/src/backend/opencl/JIT/ScalarNode.hpp b/src/backend/opencl/JIT/ScalarNode.hpp
index b172b67680..e3e269e1fd 100644
--- a/src/backend/opencl/JIT/ScalarNode.hpp
+++ b/src/backend/opencl/JIT/ScalarNode.hpp
@@ -28,76 +28,33 @@ namespace JIT
     public:
 
         ScalarNode(T val)
-            : Node(dtype_traits<T>::getName(), shortname<T>(false), 0),
+            : Node(dtype_traits<T>::getName(), shortname<T>(false), 0, {}),
               m_val(val)
         {
         }
 
-        bool isLinear(dim_t dims[4])
+        void genKerName(std::stringstream &kerStream, Node_ids ids)
         {
-            return true;
-        }
-
-        void genKerName(std::stringstream &kerStream)
-        {
-            if (m_gen_name) return;
-
             kerStream << "_" << m_name_str;
-            kerStream << std::setw(3) << std::setfill('0') << std::dec << m_id << std::dec;
-            m_gen_name = true;
+            kerStream << std::setw(3) << std::setfill('0') << std::dec << ids.id << std::dec;
         }
 
-        void genParams(std::stringstream &kerStream, bool is_linear)
+        void genParams(std::stringstream &kerStream, int id, bool is_linear)
         {
-            if (m_gen_param) return;
-            kerStream << m_type_str << " scalar" << m_id << ", " << "\n";
-            m_gen_param = true;
+            kerStream << m_type_str << " scalar" << id << ", " << "\n";
         }
 
         int setArgs(cl::Kernel &ker, int id, bool is_linear)
         {
-            if (m_set_arg) return id;
             ker.setArg(id, m_val);
-            m_set_arg = true;
             return id + 1;
         }
 
-        void genOffsets(std::stringstream &kerStream, bool is_linear)
+        void genFuncs(std::stringstream &kerStream, Node_ids ids)
         {
-            if (m_gen_offset) return;
-            m_gen_offset = true;
-        }
-
-        void genFuncs(std::stringstream &kerStream)
-        {
-            if (m_gen_func) return;
-
-            kerStream << m_type_str << " val" << m_id << " = "
-                      << "scalar" << m_id << ";"
+            kerStream << m_type_str << " val" << ids.id << " = "
+                      << "scalar" << ids.id << ";"
                       << "\n";
-
-            m_gen_func = true;
-        }
-
-        int setId(int id)
-        {
-            if (m_set_id) return id;
-            m_id = id;
-            m_set_id = true;
-            return m_id + 1;
-        }
-
-        void getInfo(unsigned &len, unsigned &buf_count, unsigned &bytes)
-        {
-            if (m_set_id) return;
-            len++;
-            m_set_id = true;
-            return;
-        }
-
-        void resetFlags()
-        {
-            if (m_set_id) resetCommonFlags();
         }
     };
 
diff --git a/src/backend/opencl/JIT/UnaryNode.hpp b/src/backend/opencl/JIT/UnaryNode.hpp
index 5b9bdbc481..6b178615ab 100644
--- a/src/backend/opencl/JIT/UnaryNode.hpp
+++ b/src/backend/opencl/JIT/UnaryNode.hpp
@@ -20,102 +20,32 @@ namespace JIT
     class UnaryNode : public Node
     {
     private:
-        std::string m_op_str;
-        Node_ptr m_child;
-        int m_op;
+        const std::string m_op_str;
+        const int m_op;
 
     public:
         UnaryNode(const char *out_type_str, const char *name_str,
                   const char *op_str,
                   Node_ptr child, int op)
-            : Node(out_type_str, name_str, child->getHeight() + 1),
+            : Node(out_type_str, name_str, child->getHeight() + 1, {child}),
               m_op_str(op_str),
-              m_child(child),
               m_op(op)
         {
         }
 
-        bool isLinear(dim_t dims[4])
+        void genKerName(std::stringstream &kerStream, Node_ids ids)
         {
-            if (!m_set_is_linear) {
-                m_linear = m_child->isLinear(dims);
-                m_set_is_linear = true;
-            }
-            return m_linear;
-        }
-
-        void genParams(std::stringstream &kerStream, bool is_linear)
-        {
-            if (m_gen_param) return;
-            if (!(m_child->isGenParam())) m_child->genParams(kerStream, is_linear);
-            m_gen_param = true;
-        }
-
-        int setArgs(cl::Kernel &ker, int id, bool is_linear)
-        {
-            if (m_set_arg) return id;
-            m_set_arg = true;
-            return m_child->setArgs(ker, id, is_linear);
-        }
-
-        void genOffsets(std::stringstream &kerStream, bool is_linear)
-        {
-            if (m_gen_offset) return;
-            if (!(m_child->isGenOffset())) m_child->genOffsets(kerStream, is_linear);
-            m_gen_offset = true;
-        }
-
-        void genKerName(std::stringstream &kerStream)
-        {
-            if (m_gen_name) return;
-            m_child->genKerName(kerStream);
-
             // Make the dec representation of enum part of the Kernel name
             kerStream << "_" << std::setw(3) << std::setfill('0') << std::dec << m_op;
-            kerStream << std::setw(3) << std::setfill('0') << std::dec << m_child->getId();
-            kerStream << std::setw(3) << std::setfill('0') << std::dec << m_id << std::dec;
-            m_gen_name = true;
+            kerStream << std::setw(3) << std::setfill('0') << std::dec << ids.child_ids[0];
+            kerStream << std::setw(3) << std::setfill('0') << std::dec << ids.id << std::dec;
         }
 
-        void genFuncs(std::stringstream &kerStream)
+        void genFuncs(std::stringstream &kerStream, Node_ids ids)
         {
-            if (m_gen_func) return;
-
-            if (!(m_child->isGenFunc())) m_child->genFuncs(kerStream);
-
-            kerStream << m_type_str << " val" << m_id << " = "
-                      << m_op_str << "(val" << m_child->getId() << ");"
+            kerStream << m_type_str << " val" << ids.id << " = "
+                      << m_op_str << "(val" << ids.child_ids[0] << ");"
                       << "\n";
-
-            m_gen_func = true;
-        }
-
-        int setId(int id)
-        {
-            if (m_set_id) return id;
-            id = m_child->setId(id);
-            m_id = id;
-            m_set_id = true;
-            return m_id + 1;
-        }
-
-        void getInfo(unsigned &len, unsigned &buf_count, unsigned &bytes)
-        {
-            if (m_set_id) return;
-
-            m_child->getInfo(len, buf_count, bytes);
-            len++;
-
-            m_set_id = true;
-            return;
-        }
-
-        void resetFlags()
-        {
-            if (m_set_id) {
-                resetCommonFlags();
-                m_child->resetFlags();
-            }
         }
     };
 
diff --git a/src/backend/opencl/jit.cpp b/src/backend/opencl/jit.cpp
index 360028d967..5442eba7f4 100644
--- a/src/backend/opencl/jit.cpp
+++ b/src/backend/opencl/jit.cpp
@@ -25,6 +25,8 @@ namespace opencl
 {
 
 using JIT::Node;
+using JIT::Node_ids;
+using JIT::Node_map_t;
 
 using cl::Buffer;
 using cl::Program;
@@ -34,8 +36,12 @@ using cl::EnqueueArgs;
 using cl::NDRange;
 using std::string;
 using std::stringstream;
+using std::vector;
 
-static string getFuncName(std::vector<Node *> nodes, bool is_linear, bool *is_double)
+static string getFuncName(const vector<Node *> &output_nodes,
+                          const vector<Node *> &full_nodes,
+                          const vector<Node_ids> &full_ids,
+                          bool is_linear, bool *is_double)
 {
     stringstream hashName;
     stringstream funcName;
@@ -46,13 +52,12 @@ static string getFuncName(std::vector<Node *> nodes, bool is_linear, bool *is_do
         funcName << "G_";
     }
 
-    int id = 0;
-    for (auto node :  nodes) {
-        funcName << "[";
-        id = node->setId(id);
-        funcName << node->getNameStr();
-        node->genKerName(funcName);
-        funcName << "]";
+    for (auto node :  output_nodes) {
+        funcName << node->getNameStr() << "_";
+    }
+
+    for (int i = 0; i < (int)full_nodes.size(); i++) {
+        full_nodes[i]->genKerName(funcName, full_ids[i]);
     }
 
     string nameStr = funcName.str();
@@ -65,7 +70,11 @@ static string getFuncName(std::vector<Node *> nodes, bool is_linear, bool *is_do
     return hashName.str();
 }
 
-static string getKernelString(string funcName, std::vector<Node *> nodes, bool is_linear)
+static string getKernelString(const string funcName,
+                              const vector<Node *> &full_nodes,
+                              const vector<Node_ids> &full_ids,
+                              const vector<int> &output_ids,
+                              bool is_linear)
 {
 
     // Common OpenCL code
@@ -118,16 +127,23 @@ static string getKernelString(string funcName, std::vector<Node *> nodes, bool i
     stringstream offsetsStream;
     stringstream opsStream;
 
-    int count  = 0;
+    for (int i = 0; i < (int)full_nodes.size(); i++) {
+        const auto &node = full_nodes[i];
+        const auto &ids_curr = full_ids[i];
+        // Generate input parameters, only needs current id
+        node->genParams(inParamStream, ids_curr.id, is_linear);
+        // Generate input offsets, only needs current id
+        node->genOffsets(offsetsStream, ids_curr.id, is_linear);
+        // Generate the core function body, needs children ids as well
+        node->genFuncs(opsStream, ids_curr);
+    }
 
-    for (auto node : nodes) {
-        int id = node->getId();
-        node->genParams(inParamStream, is_linear);
-        outParamStream << "__global " << node->getTypeStr() << " *out" << id << ", \n";
+    for (int i = 0; i < (int)output_ids.size(); i++) {
+        int id = output_ids[i];
+        // Generate output parameters
+        outParamStream << "__global " << full_nodes[id]->getTypeStr() << " *out" << id << ", \n";
+        // Generate code to write the output
         outWriteStream << "out" << id << "[idx] = " << "val" << id << ";\n";
-        node->genOffsets(offsetsStream, is_linear);
-        node->genFuncs(opsStream);
-        opsStream << "//" << ++count << std::endl << std::endl;
     }
 
     // Put various blocks into a single stream
@@ -153,16 +169,20 @@ static string getKernelString(string funcName, std::vector<Node *> nodes, bool i
     return kerStream.str();
 }
 
-static Kernel getKernel(std::vector<Node *> nodes, bool is_linear)
+static Kernel getKernel(const vector<Node *> &output_nodes,
+                        const vector<int> &output_ids,
+                        const vector<Node *> &full_nodes,
+                        const vector<Node_ids> &full_ids,
+                        const bool is_linear)
 {
     bool is_dbl = false;
-    string funcName = getFuncName(nodes, is_linear, &is_dbl);
+    string funcName = getFuncName(output_nodes, full_nodes, full_ids, is_linear, &is_dbl);
     int device = getActiveDeviceId();
 
     kc_entry_t entry = kernelCache(device, funcName);
 
     if (entry.prog==0 && entry.ker==0) {
-        string jit_ker = getKernelString(funcName, nodes, is_linear);
+        string jit_ker = getKernelString(funcName, full_nodes, full_ids, output_ids, is_linear);
 
         const char *ker_strs[] = {jit_cl, jit_ker.c_str()};
         const int ker_lens[] = {jit_cl_len, (int)jit_ker.size()};
@@ -179,7 +199,7 @@ static Kernel getKernel(std::vector<Node *> nodes, bool is_linear)
     return *entry.ker;
 }
 
-void evalNodes(std::vector<Param> &outputs, std::vector<Node *> nodes)
+void evalNodes(vector<Param> &outputs, vector<Node *> output_nodes)
 {
     if (outputs.size() == 0) return;
 
@@ -187,13 +207,25 @@ void evalNodes(std::vector<Param> &outputs, std::vector<Node *> nodes)
     //FIXME: Add assert to check if all outputs are same size?
     KParam out_info = outputs[0].info;
 
-    // Verify if all ASTs hold Linear Arrays
+    Node_map_t nodes;
+    vector<int> output_ids;
+    for (auto &node : output_nodes) {
+        node->getNodesMap(nodes);
+        output_ids.push_back(nodes[node].id);
+    }
+
+    vector<Node *> full_nodes(nodes.size());
+    vector<Node_ids> full_ids(nodes.size());
     bool is_linear = true;
-    for (auto node : nodes) {
-        is_linear &= node->isLinear(out_info.dims);
+    for (auto &map_entry : nodes) {
+        full_nodes[map_entry.second.id] = map_entry.first;
+        full_ids[map_entry.second.id] = map_entry.second;
+        is_linear &= map_entry.first->isLinear(out_info.dims);
     }
 
-    Kernel ker = getKernel(nodes, is_linear);
+    Kernel ker = getKernel(output_nodes, output_ids,
+                           full_nodes, full_ids,
+                           is_linear);
 
     uint local_0 = 1;
     uint local_1 = 1;
@@ -233,36 +265,32 @@ void evalNodes(std::vector<Param> &outputs, std::vector<Node *> nodes)
     NDRange local(local_0, local_1);
     NDRange global(global_0, global_1);
 
-    int args = 0;
-    for (auto node : nodes) {
-        args = node->setArgs(ker, args, is_linear);
+    int nargs = 0;
+    for (const auto &node : full_nodes) {
+        nargs = node->setArgs(ker, nargs, is_linear);
     }
 
     // Set output parameters
     for (auto output : outputs) {
-        ker.setArg(args, *(output.data));
-        ++args;
+        ker.setArg(nargs, *(output.data));
+        ++nargs;
     }
 
     // Set dimensions
     // All outputs are asserted to be of same size
     // Just use the size from the first output
-    ker.setArg(args + 0,  out_info);
-    ker.setArg(args + 1,  groups_0);
-    ker.setArg(args + 2,  groups_1);
-    ker.setArg(args + 3,  num_odims);
+    ker.setArg(nargs + 0,  out_info);
+    ker.setArg(nargs + 1,  groups_0);
+    ker.setArg(nargs + 2,  groups_1);
+    ker.setArg(nargs + 3,  num_odims);
 
     getQueue().enqueueNDRangeKernel(ker, cl::NullRange, global, local);
-
-    for (auto node : nodes) {
-        node->resetFlags();
-    }
 }
 
 void evalNodes(Param &out, Node *node)
 {
-    std::vector<Param>  outputs{out};
-    std::vector<Node *> nodes{node};
+    vector<Param>  outputs{out};
+    vector<Node *> nodes{node};
     return evalNodes(outputs, nodes);
 }
 

From baf27d4bfae63425ee91c05ba0c6d45799fb94de Mon Sep 17 00:00:00 2001
From: Pavan Yalamanchili <contact@pavanky.com>
Date: Thu, 16 Mar 2017 02:44:15 -0700
Subject: [PATCH 50/82] Cleaning up TNJ in CPU backend and making it thread
 safe

---
 src/backend/cpu/Array.cpp          | 74 +++++++++++++---------
 src/backend/cpu/Array.hpp          |  5 ++
 src/backend/cpu/TNJ/BinaryNode.hpp | 57 +++--------------
 src/backend/cpu/TNJ/BufferNode.hpp | 97 ++++++++++++++---------------
 src/backend/cpu/TNJ/Node.hpp       | 95 +++++++++++++----------------
 src/backend/cpu/TNJ/ScalarNode.hpp | 29 +--------
 src/backend/cpu/TNJ/UnaryNode.hpp  | 48 +++------------
 src/backend/cpu/kernel/Array.hpp   | 98 ++++++++++++++++++++++++++----
 8 files changed, 242 insertions(+), 261 deletions(-)

diff --git a/src/backend/cpu/Array.cpp b/src/backend/cpu/Array.cpp
index 96d0345f6d..781521bf41 100644
--- a/src/backend/cpu/Array.cpp
+++ b/src/backend/cpu/Array.cpp
@@ -30,18 +30,25 @@ using TNJ::Node_ptr;
 
 using af::dim4;
 
+
+template<typename T>
+Node_ptr bufferNodePtr()
+{
+    return Node_ptr(reinterpret_cast<Node *>(new BufferNode<T>()));
+}
+
 template<typename T>
 Array<T>::Array(dim4 dims):
     info(getActiveDeviceId(), dims, 0, calcStrides(dims), (af_dtype)dtype_traits<T>::af_type),
     data(memAlloc<T>(dims.elements()), memFree<T>), data_dims(dims),
-    node(), ready(true), owner(true)
+    node(bufferNodePtr<T>()), ready(true), owner(true)
 { }
 
 template<typename T>
 Array<T>::Array(dim4 dims, const T * const in_data, bool is_device, bool copy_device):
     info(getActiveDeviceId(), dims, 0, calcStrides(dims), (af_dtype)dtype_traits<T>::af_type),
     data((is_device & !copy_device) ? (T*)in_data : memAlloc<T>(dims.elements()), memFree<T>), data_dims(dims),
-    node(), ready(true), owner(true)
+    node(bufferNodePtr<T>()), ready(true), owner(true)
 {
     static_assert(std::is_standard_layout<Array<T>>::value, "Array<T> must be a standard layout type");
     static_assert(offsetof(Array<T>, info) == 0, "Array<T>::info must be the first member variable of Array<T>");
@@ -62,7 +69,7 @@ template<typename T>
 Array<T>::Array(const Array<T>& parent, const dim4 &dims, const dim_t &offset_, const dim4 &strides) :
     info(parent.getDevId(), dims, offset_, strides, (af_dtype)dtype_traits<T>::af_type),
     data(parent.getData()), data_dims(parent.getDataDims()),
-    node(),
+    node(bufferNodePtr<T>()),
     ready(true), owner(false)
 { }
 
@@ -72,7 +79,7 @@ Array<T>::Array(af::dim4 dims, af::dim4 strides, dim_t offset_,
     info(getActiveDeviceId(), dims, offset_, strides, (af_dtype)dtype_traits<T>::af_type),
     data(is_device ? (T*)in_data : memAlloc<T>(info.total()), memFree<T>),
     data_dims(dims),
-    node(),
+    node(bufferNodePtr<T>()),
     ready(true),
     owner(true)
 {
@@ -93,7 +100,7 @@ void Array<T>::eval()
 
     getQueue().enqueue(kernel::evalArray<T>, *this);
     // Reset shared_ptr
-    this->node.reset();
+    this->node = bufferNodePtr<T>();
     ready = true;
 }
 
@@ -115,12 +122,25 @@ T* Array<T>::device()
 }
 
 template<typename T>
-void evalMultiple(std::vector<Array<T>*> arrays)
+void evalMultiple(std::vector<Array<T>*> array_ptrs)
 {
-    //FIXME: implement this correctly
-    //Using fallback for now
-    for (auto array : arrays) {
-        array->eval();
+    std::vector<Array<T>> arrays;
+    bool isWorker = getQueue().is_worker();
+    for (auto &array : array_ptrs) {
+        if (array->ready) continue;
+        if (isWorker) AF_ERROR("Array not evaluated", AF_ERR_INTERNAL);
+        array->setId(getActiveDeviceId());
+        array->data = std::shared_ptr<T>(memAlloc<T>(array->elements()), memFree<T>);
+        arrays.push_back(*array);
+    }
+
+    if (arrays.size() > 0) {
+        getQueue().enqueue(kernel::evalMultiple<T>, arrays);
+        for (auto &array : array_ptrs) {
+            if (array->ready) continue;
+            array->ready = true;
+            array->node = bufferNodePtr<T>();
+        }
     }
     return;
 }
@@ -128,20 +148,16 @@ void evalMultiple(std::vector<Array<T>*> arrays)
 template<typename T>
 Node_ptr Array<T>::getNode() const
 {
-    if (!node) {
-
+    if (node->isBuffer()) {
+        BufferNode<T> *bufNode = reinterpret_cast<BufferNode<T> *>(node.get());
         unsigned bytes = this->getDataDims().elements() * sizeof(T);
-
-        BufferNode<T> *buf_node = new BufferNode<T>(data,
-                                                    bytes,
-                                                    getOffset(),
-                                                    dims().get(),
-                                                    strides().get(),
-                                                    isLinear());
-
-        const_cast<Array<T> *>(this)->node = Node_ptr(reinterpret_cast<Node *>(buf_node));
+        bufNode->setData(data,
+                         bytes,
+                         getOffset(),
+                         dims().get(),
+                         strides().get(),
+                         isLinear());
     }
-
     return node;
 }
 
@@ -198,15 +214,15 @@ createNodeArray(const dim4 &dims, Node_ptr node)
             if (lock_bytes > getMaxBytes() ||
                 lock_buffers > getMaxBuffers()) {
 
-                // Calling sync to ensure the TNJ calls below
-                // don't overwrite the same nodes being evaluated
-                // FIXME: This should ideally be JIT specific mutex
-                getQueue().sync();
+                Node *n = node.get();
 
+                TNJ::Node_map_t nodes_map;
+                n->getNodesMap(nodes_map);
                 unsigned length =0, buf_count = 0, bytes = 0;
-                Node *n = node.get();
-                n->getInfo(length, buf_count, bytes);
-                n->reset();
+                for(auto &entry : nodes_map) {
+                    Node *node = entry.first;
+                    node->getInfo(length, buf_count, bytes);
+                }
 
                 if (2 * bytes > lock_bytes) {
                     out.eval();
diff --git a/src/backend/cpu/Array.hpp b/src/backend/cpu/Array.hpp
index 65ef5f1434..c3a1286708 100644
--- a/src/backend/cpu/Array.hpp
+++ b/src/backend/cpu/Array.hpp
@@ -30,6 +30,10 @@ template<typename T> class Array;
 namespace kernel
 {
 template<typename T> void evalArray(cpu::Array<T> in);
+
+template<typename T>
+void evalMultiple(std::vector<Array<T>> arrays);
+
 }
 }
 
@@ -234,6 +238,7 @@ namespace cpu
                                           bool copy);
 
         friend void kernel::evalArray<T>(Array<T> in);
+        friend void kernel::evalMultiple<T>(std::vector<Array<T>> arrays);
 
         friend void destroyArray<T>(Array<T> *arr);
         friend void *getDevicePtr<T>(const Array<T>& arr);
diff --git a/src/backend/cpu/TNJ/BinaryNode.hpp b/src/backend/cpu/TNJ/BinaryNode.hpp
index f06581a395..78c7c6df66 100644
--- a/src/backend/cpu/TNJ/BinaryNode.hpp
+++ b/src/backend/cpu/TNJ/BinaryNode.hpp
@@ -29,68 +29,29 @@ namespace TNJ
 {
 
     template<typename To, typename Ti, af_op_t op>
-    class BinaryNode  : public Node
+    class BinaryNode  : public TNode<To>
     {
 
     protected:
-        Node_ptr m_lhs;
-        Node_ptr m_rhs;
         BinOp<To, Ti, op> m_op;
-        To m_val;
+        TNode<Ti> *m_lhs, *m_rhs;
 
     public:
         BinaryNode(Node_ptr lhs, Node_ptr rhs) :
-            Node(std::max(lhs->getHeight(), rhs->getHeight()) + 1),
-            m_lhs(lhs),
-            m_rhs(rhs),
-            m_val(0)
+            TNode<To>(0, std::max(lhs->getHeight(), rhs->getHeight()) + 1, {lhs, rhs}),
+            m_lhs(reinterpret_cast<TNode<Ti> *>(lhs.get())),
+            m_rhs(reinterpret_cast<TNode<Ti> *>(rhs.get()))
         {
         }
 
-        void *calc(int x, int y, int z, int w)
+        void calc(int x, int y, int z, int w)
         {
-            if (calcCurrent(x, y, z, w)) {
-                m_val = m_op.eval(*(Ti *)m_lhs->calc(x, y, z, w),
-                                  *(Ti *)m_rhs->calc(x, y, z, w));
-            }
-            return  (void *)&m_val;
+            this->m_val = m_op.eval(m_lhs->m_val, m_rhs->m_val);
         }
 
-        void *calc(int idx)
+        void calc(int idx)
         {
-            if (calcCurrent(idx)) {
-                m_val = m_op.eval(*(Ti *)m_lhs->calc(idx),
-                                  *(Ti *)m_rhs->calc(idx));
-            }
-            return (void *)&m_val;
-        }
-
-        void getInfo(unsigned &len, unsigned &buf_count, unsigned &bytes)
-        {
-            if (m_is_eval) return;
-
-            m_lhs->getInfo(len, buf_count, bytes);
-            m_rhs->getInfo(len, buf_count, bytes);
-            len++;
-
-            m_is_eval = true;
-            return;
-        }
-
-        void reset()
-        {
-            resetCommonFlags();
-            m_lhs->reset();
-            m_rhs->reset();
-        }
-
-        bool isLinear(const dim_t *dims)
-        {
-            if (!m_set_is_linear) {
-                m_linear = m_lhs->isLinear(dims) && m_rhs->isLinear(dims);
-                m_set_is_linear = true;
-            }
-            return m_linear;
+            this->m_val = m_op.eval(m_lhs->m_val, m_rhs->m_val);
         }
     };
 
diff --git a/src/backend/cpu/TNJ/BufferNode.hpp b/src/backend/cpu/TNJ/BufferNode.hpp
index d995314ae7..bdd7c7e700 100644
--- a/src/backend/cpu/TNJ/BufferNode.hpp
+++ b/src/backend/cpu/TNJ/BufferNode.hpp
@@ -11,7 +11,7 @@
 #include <optypes.hpp>
 #include <vector>
 #include "Node.hpp"
-
+#include <mutex>
 namespace cpu
 {
 
@@ -20,87 +20,78 @@ namespace TNJ
 
     using std::shared_ptr;
     template<typename T>
-    class BufferNode : public Node
+    class BufferNode : public TNode<T>
     {
 
     protected:
-        shared_ptr<T> ptr;
+        shared_ptr<T> m_sptr;
+        T *m_ptr;
         unsigned m_bytes;
         bool m_linear_buffer;
-        dim_t m_off;
         dim_t m_strides[4];
         dim_t m_dims[4];
-        T m_val;
+        std::once_flag m_set_data_flag;
     public:
 
-        BufferNode(shared_ptr<T> data,
-                   unsigned bytes,
-                   dim_t data_off,
-                   const dim_t *dms,
-                   const dim_t *strs,
-                   const bool is_linear) :
-            Node(0),
-            ptr(data),
-            m_bytes(bytes),
-            m_linear_buffer(is_linear),
-            m_off(data_off),
-            m_val(0)
+        BufferNode() : TNode<T>(0, 0, {})
+        {}
+
+        void setData(shared_ptr<T> data,
+                     unsigned bytes,
+                     dim_t data_off,
+                     const dim_t *dims,
+                     const dim_t *strides,
+                     const bool is_linear)
         {
-            for (int i = 0; i < 4; i++) {
-                m_strides[i] = strs[i];
-                m_dims[i] = dms[i];
-            }
+            std::call_once(m_set_data_flag,
+                           [this, data, bytes,
+                            data_off, dims, strides, is_linear]()
+                           {
+                               m_sptr = data;
+                               m_ptr = data.get() + data_off;
+                               m_bytes = bytes;
+                               m_linear_buffer = is_linear;
+                               for (int i = 0; i < 4; i++) {
+                                   m_strides[i] = strides[i];
+                                   m_dims[i] = dims[i];
+                               }
+                           });
         }
 
-        void *calc(int x, int y, int z, int w)
+        void calc(int x, int y, int z, int w)
         {
-            if (calcCurrent(x, y, z, w)) {
-                dim_t l_off = 0;
-                l_off += (w < (int)m_dims[3]) * w * m_strides[3];
-                l_off += (z < (int)m_dims[2]) * z * m_strides[2];
-                l_off += (y < (int)m_dims[1]) * y * m_strides[1];
-                l_off += (x < (int)m_dims[0]) * x;
-                m_val = *(ptr.get() + m_off + l_off);
-            }
-            return (void *)&m_val;
+            dim_t l_off = 0;
+            l_off += (w < (int)m_dims[3]) * w * m_strides[3];
+            l_off += (z < (int)m_dims[2]) * z * m_strides[2];
+            l_off += (y < (int)m_dims[1]) * y * m_strides[1];
+            l_off += (x < (int)m_dims[0]) * x;
+            this->m_val = m_ptr[l_off];
         }
 
-        void *calc(int idx)
+        void calc(int idx)
         {
-            if (calcCurrent(idx)) {
-                m_val = *(ptr.get() + idx + m_off);
-            }
-            return (void *)&m_val;
+            this->m_val = m_ptr[idx];
         }
 
         void getInfo(unsigned &len, unsigned &buf_count, unsigned &bytes)
         {
-            if (m_is_eval) return;
-
             len++;
             buf_count++;
             bytes += m_bytes;
-            m_is_eval = true;
             return;
         }
 
-        void reset()
-        {
-            resetCommonFlags();
-        }
-
         bool isLinear(const dim_t *dims)
         {
-            if (!m_set_is_linear) {
-                m_linear = m_linear_buffer &&
-                    dims[0] == m_dims[0] &&
-                    dims[1] == m_dims[1] &&
-                    dims[2] == m_dims[2] &&
-                    dims[3] == m_dims[3];
-                m_set_is_linear = true;
-            }
-            return m_linear;
+            return m_linear_buffer &&
+                dims[0] == m_dims[0] &&
+                dims[1] == m_dims[1] &&
+                dims[2] == m_dims[2] &&
+                dims[3] == m_dims[3];
         }
+
+        bool isBuffer() { return true; }
+
     };
 
 }
diff --git a/src/backend/cpu/TNJ/Node.hpp b/src/backend/cpu/TNJ/Node.hpp
index f4ed00bd5c..0d3e6c9896 100644
--- a/src/backend/cpu/TNJ/Node.hpp
+++ b/src/backend/cpu/TNJ/Node.hpp
@@ -11,6 +11,7 @@
 #include <optypes.hpp>
 #include <vector>
 #include <memory>
+#include <unordered_map>
 
 namespace cpu
 {
@@ -18,86 +19,74 @@ namespace cpu
 namespace TNJ
 {
 
+    class Node;
+    using std::shared_ptr;
+    typedef shared_ptr<Node> Node_ptr;
+
+    typedef std::unordered_map<Node *, int> Node_map_t;
+    typedef Node_map_t::iterator Node_map_iter;
+
     class Node
     {
 
     protected:
 
         const int m_height;
-        int x, y, z, w;
-        bool m_is_eval;
-        bool m_linear;
-        bool m_set_is_linear;
-
-
-        void resetCommonFlags()
-        {
-            x = -1;
-            y = -1;
-            z = -1;
-            w = -1;
-            m_is_eval = false;
-            m_linear = false;
-            m_set_is_linear = false;
-        }
-
-        bool calcCurrent(int xc)
-        {
-            bool res = (x == xc);
-            x = xc;
-            return !res;
-        }
-
-        bool calcCurrent(int xc, int yc, int zc, int wc)
-        {
-            bool res = (xc == x) && (yc == y) && (zc == z) && (wc == w);
-            x = xc;
-            y = yc;
-            z = zc;
-            w = wc;
-            return !res;
-        }
+        const std::vector<Node_ptr> m_children;
 
     public:
-        Node(const int height) :
+        Node(const int height, const std::vector<Node_ptr> children) :
             m_height(height),
-            x(-1),
-            y(-1),
-            z(-1),
-            w(-1),
-            m_is_eval(false),
-            m_linear(false),
-            m_set_is_linear(false)
+            m_children(children)
         {}
 
+        void getNodesMap(Node_map_t &node_map)
+        {
+            if (node_map.find(this) == node_map.end()) {
+                for (const auto &child : m_children) {
+                    child->getNodesMap(node_map);
+                }
+                int id = node_map.size();
+                node_map[this] = id;
+            }
+        }
+
         int getHeight() { return m_height; }
 
-        virtual void *calc(int x, int y, int z, int w)
+        virtual void calc(int x, int y, int z, int w)
         {
-            m_is_eval = true;
-            return NULL;
         }
 
-        virtual void *calc(int idx)
+        virtual void calc(int idx)
         {
-            m_is_eval = true;
-            return NULL;
         }
 
         virtual void getInfo(unsigned &len, unsigned &buf_count, unsigned &bytes)
         {
-            len = 0;
-            buf_count = 0;
-            bytes = 0;
+            len++;
         }
 
         virtual bool isLinear(const dim_t *dims) { return true; }
-        virtual void reset() { resetCommonFlags(); }
-
+        virtual bool isBuffer() { return false; }
         virtual ~Node() {}
+
+    };
+
+    template<typename T>
+    class TNode : public Node
+    {
+    public:
+        T m_val;
+    public:
+        TNode(T val, const int height, const std::vector<Node_ptr> children) :
+            Node(height, children),
+            m_val(val)
+            {
+            }
     };
 
-    typedef std::shared_ptr<Node> Node_ptr;
+    template<typename T>
+    using TNode_ptr = std::shared_ptr<TNode<T>>;
 }
 
 }
diff --git a/src/backend/cpu/TNJ/ScalarNode.hpp b/src/backend/cpu/TNJ/ScalarNode.hpp
index bda529f8d4..716c5964a9 100644
--- a/src/backend/cpu/TNJ/ScalarNode.hpp
+++ b/src/backend/cpu/TNJ/ScalarNode.hpp
@@ -19,38 +19,13 @@ namespace TNJ
 {
 
     template<typename T>
-    class ScalarNode : public Node
+    class ScalarNode : public TNode<T>
     {
 
-    protected:
-        T m_val;
-
     public:
-        ScalarNode(T val) : Node(0), m_val(val)
-        {
-        }
-
-        void *calc(int x, int y, int z, int w)
-        {
-            return (void *)(&m_val);
-        }
-
-        void *calc(int idx)
-        {
-            return (void *)&m_val;
-        }
-
-        void getInfo(unsigned &len, unsigned &buf_count, unsigned &bytes)
+        ScalarNode(T val) : TNode<T>(val, 0, {})
         {
-            if (m_is_eval) return;
-            len++;
-            m_is_eval = true;
-            return;
         }
-
-        void reset() { resetCommonFlags(); }
-
-        bool isLinear(const dim_t *dims) { return true; }
     };
 }
 
diff --git a/src/backend/cpu/TNJ/UnaryNode.hpp b/src/backend/cpu/TNJ/UnaryNode.hpp
index 047151f3f7..2f9d121c20 100644
--- a/src/backend/cpu/TNJ/UnaryNode.hpp
+++ b/src/backend/cpu/TNJ/UnaryNode.hpp
@@ -29,63 +29,31 @@ namespace TNJ
 {
 
     template<typename To, typename Ti, af_op_t op>
-    class UnaryNode  : public Node
+    class UnaryNode  : public TNode<To>
     {
 
     protected:
-        Node_ptr m_child;
         UnOp <To, Ti, op> m_op;
-        To m_val;
+        TNode<Ti> *m_child;
 
     public:
         UnaryNode(Node_ptr child) :
-            Node(child->getHeight() + 1),
-            m_child(child),
-            m_val(0)
+            TNode<To>(0, child->getHeight() + 1, {child}),
+            m_child(reinterpret_cast<TNode<Ti> *>(child.get()))
         {
         }
 
-        void *calc(int x, int y, int z, int w)
-        {
-            if (calcCurrent(x, y, z, w)) {
-                m_val = m_op.eval(*(Ti *)m_child->calc(x, y, z, w));
-            }
-            return (void *)(&m_val);
-        }
 
-        void *calc(int idx)
+        void calc(int x, int y, int z, int w)
         {
-            if (calcCurrent(idx)) {
-                m_val = m_op.eval(*(Ti *)m_child->calc(idx));
-            }
-            return (void *)&m_val;
+            this->m_val = m_op.eval(m_child->m_val);
         }
 
-        void getInfo(unsigned &len, unsigned &buf_count, unsigned &bytes)
+        void calc(int idx)
         {
-            if (m_is_eval) return;
-
-            m_child->getInfo(len, buf_count, bytes);
-            len++;
-
-            m_is_eval = true;
-            return;
+            this->m_val = m_op.eval(m_child->m_val);
         }
 
-        void reset()
-        {
-            resetCommonFlags();
-            m_child->reset();
-        }
-
-        bool isLinear(const dim_t *dims)
-        {
-            if (!m_set_is_linear) {
-                m_linear = m_child->isLinear(dims);
-                m_set_is_linear = true;
-            }
-            return m_linear;
-        }
     };
 
 }
diff --git a/src/backend/cpu/kernel/Array.hpp b/src/backend/cpu/kernel/Array.hpp
index 3c4a736298..7b7504064c 100644
--- a/src/backend/cpu/kernel/Array.hpp
+++ b/src/backend/cpu/kernel/Array.hpp
@@ -10,6 +10,8 @@
 #pragma once
 #include <Array.hpp>
 #include <platform.hpp>
+#include <TNJ/Node.hpp>
+#include <vector>
 
 namespace cpu
 {
@@ -17,20 +19,39 @@ namespace kernel
 {
 
 template<typename T>
-void evalArray(Array<T> in)
+void evalMultiple(std::vector<Array<T>> arrays)
 {
-    in.setId(cpu::getActiveDeviceId());
-    T *ptr = in.data.get();
+    af::dim4 odims = arrays[0].dims();
+    af::dim4 ostrs = arrays[0].strides();
 
-    af::dim4 odims = in.dims();
-    af::dim4 ostrs = in.strides();
+    int devId = cpu::getActiveDeviceId();
+    TNJ::Node_map_t nodes;
+    std::vector<T *> ptrs;
+    std::vector<TNJ::TNode<T> *> output_nodes;
 
-    bool is_linear = in.node->isLinear(odims.get());
+    for (auto &arr : arrays) {
+        arr.setId(devId);
+        ptrs.push_back(arr.data.get());
+        output_nodes.push_back(reinterpret_cast<TNJ::TNode<T> *>(arr.node.get()));
+        arr.node->getNodesMap(nodes);
+    }
+
+    bool is_linear = true;
+    std::vector<TNJ::Node *> full_nodes(nodes.size());
+    for(const auto &map_entry : nodes) {
+        full_nodes[map_entry.second] = map_entry.first;
+        is_linear &= map_entry.first->isLinear(odims.get());
+    }
 
     if (is_linear) {
-        int num = in.elements();
+        int num = arrays[0].elements();
         for (int i = 0; i < num; i++) {
-            ptr[i] = *(T *)in.node->calc(i);
+            for (int n = 0; n < (int)full_nodes.size(); n++) {
+                full_nodes[n]->calc(i);
+            }
+            for (int n = 0; n < (int)output_nodes.size(); n++) {
+                ptrs[n][i] = output_nodes[n]->m_val;
+            }
         }
     } else {
         for (int w = 0; w < (int)odims[3]; w++) {
@@ -45,15 +66,70 @@ void evalArray(Array<T> in)
                     for (int x = 0; x < (int)odims[0]; x++) {
                         dim_t id = x + offy;
 
-                        ptr[id] = *(T *)in.node->calc(x, y, z, w);
+                        for (int n = 0; n < (int)full_nodes.size(); n++) {
+                            full_nodes[n]->calc(x, y, z, w);
+                        }
+                        for (int n = 0; n < (int)output_nodes.size(); n++) {
+                            ptrs[n][id] = output_nodes[n]->m_val;
+                        }
                     }
                 }
             }
         }
     }
+}
+
+template<typename T>
+void evalArray(Array<T> arr)
+{
+    arr.setId(cpu::getActiveDeviceId());
+    T *ptr = arr.data.get();
+
+    af::dim4 odims = arr.dims();
+    af::dim4 ostrs = arr.strides();
+
+    TNJ::Node_map_t nodes;
+    arr.node->getNodesMap(nodes);
+
+    bool is_linear = true;
+    std::vector<TNJ::Node *> full_nodes(nodes.size());
+
+    for(const auto &map_entry : nodes) {
+        full_nodes[map_entry.second] = map_entry.first;
+        is_linear &= map_entry.first->isLinear(odims.get());
+    }
+
+    TNJ::TNode<T> *output_node = reinterpret_cast<TNJ::TNode<T> *>(full_nodes.back());
+    if (is_linear) {
+        int num = arr.elements();
+        for (int i = 0; i < num; i++) {
+            for (int n = 0; n < (int)full_nodes.size(); n++) {
+                full_nodes[n]->calc(i);
+            }
+            ptr[i] = output_node->m_val;
+        }
+    } else {
+        for (int w = 0; w < (int)odims[3]; w++) {
+            dim_t offw = w * ostrs[3];
+
+            for (int z = 0; z < (int)odims[2]; z++) {
+                dim_t offz = z * ostrs[2] + offw;
+
+                for (int y = 0; y < (int)odims[1]; y++) {
+                    dim_t offy = y * ostrs[1] + offz;
 
-    // Reset TNJ flags
-    in.node->reset();
+                    for (int x = 0; x < (int)odims[0]; x++) {
+                        dim_t id = x + offy;
+
+                        for (int n = 0; n < (int)full_nodes.size(); n++) {
+                            full_nodes[n]->calc(x, y, z, w);
+                        }
+                        ptr[id] = output_node->m_val;
+                    }
+                }
+            }
+        }
+    }
 }
 
 }

From 96e9ff1a98470873d801bc33866c9b89440ab726 Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Mon, 20 Mar 2017 11:57:28 +0530
Subject: [PATCH 51/82] Re-enable per thread device unit test

---
 test/threading.cpp | 164 ++++++++++++++++++++++-----------------------
 1 file changed, 82 insertions(+), 82 deletions(-)

diff --git a/test/threading.cpp b/test/threading.cpp
index a1e084f227..52ce7f6cff 100644
--- a/test/threading.cpp
+++ b/test/threading.cpp
@@ -57,88 +57,88 @@ void morphTest(const array input, const array mask, const bool isDilation,
               << diff.count() << " s\n";
 }
 
-//TEST(Threading, SetPerThreadActiveDevice)
-//{
-//    if (noImageIOTests()) return;
-//
-//    vector<bool> isDilationFlags;
-//    vector<bool> isColorFlags;
-//    vector<string> files;
-//
-//    files.push_back( string(TEST_DIR "/morph/gray.test") );
-//    isDilationFlags.push_back(true);
-//    isColorFlags.push_back(false);
-//
-//    files.push_back( string(TEST_DIR "/morph/color.test") );
-//    isDilationFlags.push_back(false);
-//    isColorFlags.push_back(true);
-//
-//    vector<std::thread> tests;
-//    unsigned totalTestCount = 0;
-//
-//    auto start = std::chrono::high_resolution_clock::now();
-//
-//    for(size_t pos = 0; pos<files.size(); ++pos)
-//    {
-//        const bool isDilation = isDilationFlags[pos];
-//        const bool isColor    = isColorFlags[pos];
-//
-//        vector<dim4>    inDims;
-//        vector<string>  inFiles;
-//        vector<dim_t>   outSizes;
-//        vector<string>  outFiles;
-//
-//        readImageTests(files[pos], inDims, inFiles, outSizes, outFiles);
-//
-//        const unsigned testCount = inDims.size();
-//
-//        const dim4 maskdims(3,3,1,1);
-//
-//        for (size_t testId=0; testId<testCount; ++testId)
-//        {
-//            int trgtDeviceId = totalTestCount % af::getDeviceCount();
-//
-//            //prefix full path to image file names
-//            inFiles[testId].insert(0,string(TEST_DIR "/morph/"));
-//            outFiles[testId].insert(0,string(TEST_DIR "/morph/"));
-//
-//            af::setDevice(trgtDeviceId);
-//
-//            const array mask = constant(1.0, maskdims);
-//
-//            array input= loadImage(inFiles[testId].c_str(), isColor);
-//            array gold = loadImage(outFiles[testId].c_str(), isColor);
-//
-//            //Push the new test as a new thread of execution
-//            tests.emplace_back(morphTest, input, mask, isDilation, gold, trgtDeviceId);
-//
-//            std::cout<<"morph test launched with the following params on device ("
-//                     <<trgtDeviceId<<"):"<<std::endl;
-//            std::cout<<"\t Input image dims: "<<input.dims()<<std::endl;
-//            std::cout<<"\t Mask dims: "<<mask.dims()<<std::endl;
-//            std::cout<<"\t IsDilation : "<< (isDilation ? "True" : "False") <<std::endl;
-//
-//            totalTestCount++;
-//        }
-//    }
-//    std::cout<< std::endl << "Waiting for results ..." << std::endl << std::endl;
-//
-//    for (size_t testId=0; testId<tests.size(); ++testId)
-//    {
-//        if (tests[testId].joinable()) {
-//            std::cout<<"Attempting join for test ..."<<testId<<std::endl;
-//            tests[testId].join();
-//            std::cout<<"test "<< testId <<" completed." << std::endl;
-//        }
-//        std::cout<<std::endl;
-//    }
-//
-//    auto end = std::chrono::high_resolution_clock::now();
-//
-//    std::chrono::duration<double> diff = end - start;
-//
-//    std::cout << "Total time taken for test : " << diff.count() << " s\n";
-//}
+TEST(Threading, SetPerThreadActiveDevice)
+{
+    if (noImageIOTests()) return;
+
+    vector<bool> isDilationFlags;
+    vector<bool> isColorFlags;
+    vector<string> files;
+
+    files.push_back( string(TEST_DIR "/morph/gray.test") );
+    isDilationFlags.push_back(true);
+    isColorFlags.push_back(false);
+
+    files.push_back( string(TEST_DIR "/morph/color.test") );
+    isDilationFlags.push_back(false);
+    isColorFlags.push_back(true);
+
+    vector<std::thread> tests;
+    unsigned totalTestCount = 0;
+
+    auto start = std::chrono::high_resolution_clock::now();
+
+    for(size_t pos = 0; pos<files.size(); ++pos)
+    {
+        const bool isDilation = isDilationFlags[pos];
+        const bool isColor    = isColorFlags[pos];
+
+        vector<dim4>    inDims;
+        vector<string>  inFiles;
+        vector<dim_t>   outSizes;
+        vector<string>  outFiles;
+
+        readImageTests(files[pos], inDims, inFiles, outSizes, outFiles);
+
+        const unsigned testCount = inDims.size();
+
+        const dim4 maskdims(3,3,1,1);
+
+        for (size_t testId=0; testId<testCount; ++testId)
+        {
+            int trgtDeviceId = totalTestCount % af::getDeviceCount();
+
+            //prefix full path to image file names
+            inFiles[testId].insert(0,string(TEST_DIR "/morph/"));
+            outFiles[testId].insert(0,string(TEST_DIR "/morph/"));
+
+            af::setDevice(trgtDeviceId);
+
+            const array mask = constant(1.0, maskdims);
+
+            array input= loadImage(inFiles[testId].c_str(), isColor);
+            array gold = loadImage(outFiles[testId].c_str(), isColor);
+
+            //Push the new test as a new thread of execution
+            tests.emplace_back(morphTest, input, mask, isDilation, gold, trgtDeviceId);
+
+            std::cout<<"morph test launched with the following params on device ("
+                     <<trgtDeviceId<<"):"<<std::endl;
+            std::cout<<"\t Input image dims: "<<input.dims()<<std::endl;
+            std::cout<<"\t Mask dims: "<<mask.dims()<<std::endl;
+            std::cout<<"\t IsDilation : "<< (isDilation ? "True" : "False") <<std::endl;
+
+            totalTestCount++;
+        }
+    }
+    std::cout<< std::endl << "Waiting for results ..." << std::endl << std::endl;
+
+    for (size_t testId=0; testId<tests.size(); ++testId)
+    {
+        if (tests[testId].joinable()) {
+            std::cout<<"Attempting join for test ..."<<testId<<std::endl;
+            tests[testId].join();
+            std::cout<<"test "<< testId <<" completed." << std::endl;
+        }
+        std::cout<<std::endl;
+    }
+
+    auto end = std::chrono::high_resolution_clock::now();
+
+    std::chrono::duration<double> diff = end - start;
+
+    std::cout << "Total time taken for test : " << diff.count() << " s\n";
+}
 
 enum ArithOp
 {

From ee6591eb3e2e5a9c5a541724f93076c9d56adb3f Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Mon, 20 Mar 2017 11:58:30 +0530
Subject: [PATCH 52/82] Remove c++11 flag for basic_c test

---
 CMakeLists.txt | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 6972b7d628..a575b68f97 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -136,7 +136,9 @@ INCLUDE_DIRECTORIES(BEFORE
     )
 
 IF(${UNIX})
-    ADD_DEFINITIONS(-Wall -std=c++11 -fvisibility=hidden)
+    ADD_DEFINITIONS(-Wall -fvisibility=hidden)
+    SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
+
     IF(${WITH_COVERAGE})
         SET(CMAKE_CXX_FLAGS             "-fprofile-arcs -ftest-coverage")
         SET(CMAKE_EXE_LINKER_FLAGS      "-fprofile-arcs -ftest-coverage")

From 1e9030a138c149b9176eaa71e1730b9a7fde67f9 Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Tue, 21 Mar 2017 18:55:19 +0530
Subject: [PATCH 53/82] Add c++11 flag for threading test alone

Earlier commit broke CUDA builds on build server though the local
build on Ubuntu 16.04 compiled fine.
---
 CMakeLists.txt                  | 3 +--
 src/backend/cuda/CMakeLists.txt | 2 +-
 test/CMakeLists.txt             | 6 ++++++
 3 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index a575b68f97..3ffe04c032 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -136,8 +136,7 @@ INCLUDE_DIRECTORIES(BEFORE
     )
 
 IF(${UNIX})
-    ADD_DEFINITIONS(-Wall -fvisibility=hidden)
-    SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
+    ADD_DEFINITIONS(-std=c++11 -Wall -fvisibility=hidden)
 
     IF(${WITH_COVERAGE})
         SET(CMAKE_CXX_FLAGS             "-fprofile-arcs -ftest-coverage")
diff --git a/src/backend/cuda/CMakeLists.txt b/src/backend/cuda/CMakeLists.txt
index 46f86dc566..e45558deed 100644
--- a/src/backend/cuda/CMakeLists.txt
+++ b/src/backend/cuda/CMakeLists.txt
@@ -418,7 +418,7 @@ MY_CUDA_ADD_LIBRARY(afcuda SHARED
                 ${thrust_sort_by_key_sources}
                 ${scan_by_key_sources}
                 OPTIONS ${CUDA_GENERATE_CODE}
-                #These flags enable C++11 and disable invalid offsetof warning
+                #disable invalid offsetof warning
                 -std=c++11 -Xcudafe "--diag_suppress=1427")
 
 ADD_DEPENDENCIES(afcuda ${ptx_targets})
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index b1023d1e14..031af068a3 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -8,6 +8,8 @@ FIND_PACKAGE(OpenCL QUIET)
 
 OPTION(BUILD_SINGLE_TEST_FILE "Build tests in a single file" OFF)
 
+REMOVE_DEFINITIONS(-std=c++11)
+
 # If the tests are not being built at the same time as ArrayFire,
 # we need to first find the ArrayFire library
 IF(TARGET afcpu OR TARGET afcuda OR TARGET afopencl OR TARGET af)
@@ -118,6 +120,10 @@ MACRO(CREATE_TESTS BACKEND AFLIBNAME GTEST_LIBS OTHER_LIBS)
           COMPILE_FLAGS -DAF_${DEF_NAME}
           FOLDER "Tests/${BACKEND}")
 
+        IF (${FNAME} STREQUAL "threading")
+            SET_TARGET_PROPERTIES(${TEST_NAME} PROPERTIES COMPILE_OPTIONS -std=c++11)
+        ENDIF()
+
         IF(TEST_LINK_FLAGS)
           SET_TARGET_PROPERTIES(${TEST_NAME} PROPERTIES LINK_FLAGS ${TEST_LINK_FLAGS})
         ENDIF(TEST_LINK_FLAGS)

From 8084be352f11aa9abf62d3ad3207529cebe660c5 Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Thu, 30 Mar 2017 17:14:37 +0530
Subject: [PATCH 54/82] Unit test for memory management in multi-threaded
 scenario

---
 test/threading.cpp | 145 ++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 143 insertions(+), 2 deletions(-)

diff --git a/test/threading.cpp b/test/threading.cpp
index 52ce7f6cff..7978cb960c 100644
--- a/test/threading.cpp
+++ b/test/threading.cpp
@@ -19,6 +19,8 @@ using namespace af;
 using std::vector;
 using std::string;
 
+static const int THREAD_COUNT = 32;
+
 #if defined(AF_CPU)
 static const unsigned ITERATION_COUNT = 10;
 #else
@@ -174,7 +176,7 @@ TEST(Threading, SimultaneousRead)
 
     vector<std::thread> tests;
 
-    for (int t=0; t<32; ++t)
+    for (int t=0; t<THREAD_COUNT; ++t)
     {
         ArithOp op;
         float outValue;
@@ -189,7 +191,146 @@ TEST(Threading, SimultaneousRead)
         tests.emplace_back(calc, op, A, B, outValue);
     }
 
-    for (int t=0; t<32; ++t)
+    for (int t=0; t<THREAD_COUNT; ++t)
         if (tests[t].joinable())
             tests[t].join();
 }
+
+int nextTargetDeviceId()
+{
+    static int nextId = 0;
+    return nextId++;
+}
+
+static void cleanSlate()
+{
+    const size_t step_bytes = 1024;
+
+    size_t alloc_bytes, alloc_buffers;
+    size_t lock_bytes, lock_buffers;
+
+    af::deviceGC();
+
+    af::deviceMemInfo(&alloc_bytes, &alloc_buffers,
+                      &lock_bytes, &lock_buffers);
+
+    ASSERT_EQ(alloc_buffers, 0u);
+    ASSERT_EQ(lock_buffers, 0u);
+    ASSERT_EQ(alloc_bytes, 0u);
+    ASSERT_EQ(lock_bytes, 0u);
+
+    af::setMemStepSize(step_bytes);
+
+    ASSERT_EQ(af::getMemStepSize(), step_bytes);
+}
+
+void doubleAllocationTest()
+{
+    af::setDevice(0);
+
+    af::array a = randu(5, 5);
+
+    for (int i = 0; i < 100; ++i)
+    {
+        a = randu(5, 5);
+        std::this_thread::sleep_for(std::chrono::milliseconds(10));
+    }
+}
+
+TEST(Threading, MemoryManagement_Double_Alloc)
+{
+    cleanSlate(); // Clean up everything done so far
+
+    vector<std::thread> tests;
+
+    for (int t=0; t<THREAD_COUNT; ++t)
+        tests.emplace_back(doubleAllocationTest);
+
+    for (int t=0; t<THREAD_COUNT; ++t)
+        if (tests[t].joinable())
+            tests[t].join();
+
+    size_t alloc_bytes, alloc_buffers;
+    size_t lock_bytes, lock_buffers;
+
+    af::deviceMemInfo(&alloc_bytes, &alloc_buffers,
+            &lock_bytes, &lock_buffers);
+
+    ASSERT_EQ( lock_buffers,     0u);
+    ASSERT_EQ(  lock_bytes,      0u);
+    ASSERT_LE(alloc_buffers,    64u);
+    ASSERT_GT(alloc_buffers,    32u);
+    ASSERT_LE(  alloc_bytes, 65536u);
+    ASSERT_GT(  alloc_bytes, 32768u);
+}
+
+void singleAllocationTest()
+{
+    af::setDevice(0);
+
+    for (int i = 0; i < 100; ++i)
+    {
+        af::array a = af::randu(5, 5);
+        std::this_thread::sleep_for(std::chrono::milliseconds(10));
+    }
+}
+
+TEST(Threading, MemoryManagement_Single_Alloc)
+{
+    cleanSlate(); // Clean up everything done so far
+
+    vector<std::thread> tests;
+
+    for (int t=0; t<THREAD_COUNT; ++t)
+        tests.emplace_back(singleAllocationTest);
+
+    for (int t=0; t<THREAD_COUNT; ++t)
+        if (tests[t].joinable())
+            tests[t].join();
+
+    size_t alloc_bytes, alloc_buffers;
+    size_t lock_bytes, lock_buffers;
+
+    af::deviceMemInfo(&alloc_bytes, &alloc_buffers,
+            &lock_bytes, &lock_buffers);
+
+    ASSERT_EQ( lock_buffers,     0u);
+    ASSERT_EQ(  lock_bytes,      0u);
+    ASSERT_LE(alloc_buffers,    32u);
+    ASSERT_GT(alloc_buffers,     0u);
+    ASSERT_LE(  alloc_bytes, 32768u);
+    ASSERT_GT(  alloc_bytes,     0u);
+}
+
+void jitAllocationTest()
+{
+    af::setDevice(0);
+
+    for (int i = 0; i < 100; ++i)
+        af::array a = af::constant(1, 5, 5);
+}
+
+TEST(Threading, MemoryManagement_JIT_Node)
+{
+    cleanSlate(); // Clean up everything done so far
+
+    vector<std::thread> tests;
+
+    for (int t=0; t<THREAD_COUNT; ++t)
+        tests.emplace_back(jitAllocationTest);
+
+    for (int t=0; t<THREAD_COUNT; ++t)
+        if (tests[t].joinable())
+            tests[t].join();
+
+    size_t alloc_bytes, alloc_buffers;
+    size_t lock_bytes, lock_buffers;
+
+    af::deviceMemInfo(&alloc_bytes, &alloc_buffers,
+            &lock_bytes, &lock_buffers);
+
+    ASSERT_EQ(alloc_buffers,     0u);
+    ASSERT_EQ( lock_buffers,     0u);
+    ASSERT_EQ(  alloc_bytes,     0u);
+    ASSERT_EQ(  lock_bytes,      0u);
+}

From 89340b63c3700584a7d99d9c129d47f8620cb16c Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Fri, 31 Mar 2017 15:32:11 +0530
Subject: [PATCH 55/82] FFT multi-threaded unit tests

---
 test/threading.cpp | 145 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 145 insertions(+)

diff --git a/test/threading.cpp b/test/threading.cpp
index 7978cb960c..5f8ce9bee0 100644
--- a/test/threading.cpp
+++ b/test/threading.cpp
@@ -334,3 +334,148 @@ TEST(Threading, MemoryManagement_JIT_Node)
     ASSERT_EQ(  alloc_bytes,     0u);
     ASSERT_EQ(  lock_bytes,      0u);
 }
+
+template<typename inType, typename outType, bool isInverse>
+void fftTest(int targetDevice, string pTestFile, dim_t pad0=0, dim_t pad1=0, dim_t pad2=0)
+{
+    if (noDoubleTests<inType>()) return;
+    if (noDoubleTests<outType>()) return;
+
+    vector<af::dim4>        numDims;
+    vector<vector<inType> >       in;
+    vector<vector<outType> >   tests;
+
+    readTestsFromFile<inType, outType>(pTestFile, numDims, in, tests);
+
+    af::dim4 dims       = numDims[0];
+    af_array outArray   = 0;
+    af_array inArray    = 0;
+
+    ASSERT_EQ(AF_SUCCESS, af_set_device(targetDevice));
+
+    ASSERT_EQ(AF_SUCCESS, af_create_array(&inArray, &(in[0].front()),
+                dims.ndims(), dims.get(), (af_dtype)af::dtype_traits<inType>::af_type));
+
+    if (isInverse){
+        switch (dims.ndims()) {
+            case 1 : ASSERT_EQ(AF_SUCCESS, af_ifft (&outArray, inArray, 1.0, pad0));              break;
+            case 2 : ASSERT_EQ(AF_SUCCESS, af_ifft2(&outArray, inArray, 1.0, pad0, pad1));        break;
+            case 3 : ASSERT_EQ(AF_SUCCESS, af_ifft3(&outArray, inArray, 1.0, pad0, pad1, pad2));  break;
+            default: throw std::runtime_error("This error shouldn't happen, pls check");
+        }
+    } else {
+        switch(dims.ndims()) {
+            case 1 : ASSERT_EQ(AF_SUCCESS, af_fft (&outArray, inArray, 1.0, pad0));               break;
+            case 2 : ASSERT_EQ(AF_SUCCESS, af_fft2(&outArray, inArray, 1.0, pad0, pad1));         break;
+            case 3 : ASSERT_EQ(AF_SUCCESS, af_fft3(&outArray, inArray, 1.0, pad0, pad1, pad2));   break;
+            default: throw std::runtime_error("This error shouldn't happen, pls check");
+        }
+    }
+
+    size_t out_size = tests[0].size();
+    outType *outData= new outType[out_size];
+    ASSERT_EQ(AF_SUCCESS, af_get_data_ptr((void*)outData, outArray));
+
+    vector<outType> goldBar(tests[0].begin(), tests[0].end());
+
+    size_t test_size = 0;
+    switch(dims.ndims()) {
+        case 1  : test_size = dims[0]/2+1;                       break;
+        case 2  : test_size = dims[1] * (dims[0]/2+1);           break;
+        case 3  : test_size = dims[2] * dims[1] * (dims[0]/2+1); break;
+        default : test_size = dims[0]/2+1;                       break;
+    }
+    outType output_scale = (outType)(isInverse ? test_size : 1);
+    for (size_t elIter=0; elIter<test_size; ++elIter) {
+        bool isUnderTolerance = abs(goldBar[elIter]-outData[elIter])<0.001;
+        ASSERT_EQ(true, isUnderTolerance)<<
+            "Expected value="<<goldBar[elIter] <<"\t Actual Value="<<
+            (output_scale*outData[elIter]) << " at: " << elIter <<
+            " from thread: "<< std::this_thread::get_id() << std::endl;
+    }
+
+    // cleanup
+    delete[] outData;
+    ASSERT_EQ(AF_SUCCESS, af_release_array(inArray));
+    ASSERT_EQ(AF_SUCCESS, af_release_array(outArray));
+}
+
+#define INSTANTIATE_TEST(func, name, is_inverse, in_t, out_t, file)                         \
+    {                                                                                       \
+        int targetDevice = nextTargetDeviceId() % numDevices;                               \
+        tests.emplace_back(fftTest<in_t, out_t, is_inverse>, targetDevice, file, 0, 0, 0);  \
+    }
+
+#define INSTANTIATE_TEST_TP(func, name, is_inverse, in_t, out_t, file, p0, p1)              \
+    {                                                                                       \
+        int targetDevice = nextTargetDeviceId() % numDevices;                               \
+        tests.emplace_back(fftTest<in_t, out_t, is_inverse>, targetDevice, file, p0, p1, 0);\
+    }
+
+#if !defined(AF_OPENCL)
+/// OpenCL backend tests seem to be failing even when
+/// each thead has it's own plan cache(thread_local).
+/// The issue seems to present itself randomly in the form of crashes,
+/// garbage values.
+TEST(Threading, FFT)
+{
+    vector<std::thread> tests;
+
+    int numDevices = 1;
+    ASSERT_EQ(AF_SUCCESS, af_get_device_count(&numDevices));
+
+    // Real to complex transforms
+    INSTANTIATE_TEST(fft ,  R2C_Float, false,  float,  cfloat, string(TEST_DIR"/signal/fft_r2c.test") );
+    INSTANTIATE_TEST(fft , R2C_Double, false, double, cdouble, string(TEST_DIR"/signal/fft_r2c.test") );
+    INSTANTIATE_TEST(fft2,  R2C_Float, false,  float,  cfloat, string(TEST_DIR"/signal/fft2_r2c.test"));
+    INSTANTIATE_TEST(fft2, R2C_Double, false, double, cdouble, string(TEST_DIR"/signal/fft2_r2c.test"));
+    INSTANTIATE_TEST(fft3,  R2C_Float, false,  float,  cfloat, string(TEST_DIR"/signal/fft3_r2c.test"));
+    INSTANTIATE_TEST(fft3, R2C_Double, false, double, cdouble, string(TEST_DIR"/signal/fft3_r2c.test"));
+
+    // complex to complex transforms
+    INSTANTIATE_TEST(fft ,  C2C_Float, false,  cfloat,  cfloat, string(TEST_DIR"/signal/fft_c2c.test") );
+    INSTANTIATE_TEST(fft , C2C_Double, false, cdouble, cdouble, string(TEST_DIR"/signal/fft_c2c.test") );
+    INSTANTIATE_TEST(fft2,  C2C_Float, false,  cfloat,  cfloat, string(TEST_DIR"/signal/fft2_c2c.test"));
+    INSTANTIATE_TEST(fft2, C2C_Double, false, cdouble, cdouble, string(TEST_DIR"/signal/fft2_c2c.test"));
+    INSTANTIATE_TEST(fft3,  C2C_Float, false,  cfloat,  cfloat, string(TEST_DIR"/signal/fft3_c2c.test"));
+    INSTANTIATE_TEST(fft3, C2C_Double, false, cdouble, cdouble, string(TEST_DIR"/signal/fft3_c2c.test"));
+
+    // Factors 7, 11, 13
+    INSTANTIATE_TEST(fft , R2C_Float_7_11_13 , false, float  , cfloat , string(TEST_DIR"/signal/fft_r2c_7_11_13.test") );
+    INSTANTIATE_TEST(fft , R2C_Double_7_11_13, false, double , cdouble, string(TEST_DIR"/signal/fft_r2c_7_11_13.test") );
+    INSTANTIATE_TEST(fft2, R2C_Float_7_11_13 , false, float  , cfloat , string(TEST_DIR"/signal/fft2_r2c_7_11_13.test") );
+    INSTANTIATE_TEST(fft2, R2C_Double_7_11_13, false, double , cdouble, string(TEST_DIR"/signal/fft2_r2c_7_11_13.test") );
+    INSTANTIATE_TEST(fft3, R2C_Float_7_11_13 , false, float  , cfloat , string(TEST_DIR"/signal/fft3_r2c_7_11_13.test") );
+    INSTANTIATE_TEST(fft3, R2C_Double_7_11_13, false, double , cdouble, string(TEST_DIR"/signal/fft3_r2c_7_11_13.test") );
+
+    INSTANTIATE_TEST(fft , C2C_Float_7_11_13 , false, cfloat  , cfloat , string(TEST_DIR"/signal/fft_c2c_7_11_13.test") );
+    INSTANTIATE_TEST(fft , C2C_Double_7_11_13, false, cdouble , cdouble, string(TEST_DIR"/signal/fft_c2c_7_11_13.test") );
+    INSTANTIATE_TEST(fft2, C2C_Float_7_11_13 , false, cfloat  , cfloat , string(TEST_DIR"/signal/fft2_c2c_7_11_13.test") );
+    INSTANTIATE_TEST(fft2, C2C_Double_7_11_13, false, cdouble , cdouble, string(TEST_DIR"/signal/fft2_c2c_7_11_13.test") );
+    INSTANTIATE_TEST(fft3, C2C_Float_7_11_13 , false, cfloat  , cfloat , string(TEST_DIR"/signal/fft3_c2c_7_11_13.test") );
+    INSTANTIATE_TEST(fft3, C2C_Double_7_11_13, false, cdouble , cdouble, string(TEST_DIR"/signal/fft3_c2c_7_11_13.test") );
+
+    // transforms on padded and truncated arrays
+    INSTANTIATE_TEST_TP(fft2,  R2C_Float_Trunc, false,  float,  cfloat, string(TEST_DIR"/signal/fft2_r2c_trunc.test"), 16, 16);
+    INSTANTIATE_TEST_TP(fft2, R2C_Double_Trunc, false, double, cdouble, string(TEST_DIR"/signal/fft2_r2c_trunc.test"), 16, 16);
+
+    INSTANTIATE_TEST_TP(fft2,  C2C_Float_Pad, false,  cfloat,  cfloat, string(TEST_DIR"/signal/fft2_c2c_pad.test"), 16, 16);
+    INSTANTIATE_TEST_TP(fft2, C2C_Double_Pad, false, cdouble, cdouble, string(TEST_DIR"/signal/fft2_c2c_pad.test"), 16, 16);
+
+    // inverse transforms
+    // complex to complex transforms
+    INSTANTIATE_TEST(ifft ,  C2C_Float, true,  cfloat,  cfloat, string(TEST_DIR"/signal/ifft_c2c.test") );
+    INSTANTIATE_TEST(ifft , C2C_Double, true, cdouble, cdouble, string(TEST_DIR"/signal/ifft_c2c.test") );
+    INSTANTIATE_TEST(ifft2,  C2C_Float, true,  cfloat,  cfloat, string(TEST_DIR"/signal/ifft2_c2c.test"));
+    INSTANTIATE_TEST(ifft2, C2C_Double, true, cdouble, cdouble, string(TEST_DIR"/signal/ifft2_c2c.test"));
+    INSTANTIATE_TEST(ifft3,  C2C_Float, true,  cfloat,  cfloat, string(TEST_DIR"/signal/ifft3_c2c.test"));
+    INSTANTIATE_TEST(ifft3, C2C_Double, true, cdouble, cdouble, string(TEST_DIR"/signal/ifft3_c2c.test"));
+
+    for (size_t testId=0; testId<tests.size(); ++testId)
+    {
+        if (tests[testId].joinable()) {
+            tests[testId].join();
+        }
+    }
+}
+#endif

From b34b8038a6c36287a9268ebb0fb11e8611ff5b3d Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Sun, 2 Apr 2017 07:10:05 +0530
Subject: [PATCH 56/82] blas multi-threaded unit tests

---
 test/threading.cpp | 95 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 95 insertions(+)

diff --git a/test/threading.cpp b/test/threading.cpp
index 5f8ce9bee0..16b0314ef4 100644
--- a/test/threading.cpp
+++ b/test/threading.cpp
@@ -13,6 +13,7 @@
 #include <testHelpers.hpp>
 #include <thread>
 #include <chrono>
+#include <iterator>
 
 using namespace af;
 
@@ -479,3 +480,97 @@ TEST(Threading, FFT)
     }
 }
 #endif
+
+template<typename T, bool isBVector>
+void cppMatMulCheck(int targetDevice, string TestFile)
+{
+    if (noDoubleTests<T>()) return;
+
+    using std::vector;
+    vector<af::dim4> numDims;
+
+    vector<vector<T> > hData;
+    vector<vector<T> > tests;
+    readTests<T,T,int>(TestFile, numDims, hData, tests);
+
+    af::setDevice(targetDevice);
+
+    af::array a(numDims[0], &hData[0].front());
+    af::array b(numDims[1], &hData[1].front());
+
+    af::dim4 atdims = numDims[0];
+    {
+        dim_t f  =    atdims[0];
+        atdims[0]   =    atdims[1];
+        atdims[1]   =    f;
+    }
+    af::dim4 btdims = numDims[1];
+    {
+        dim_t f = btdims[0];
+        btdims[0] = btdims[1];
+        btdims[1] = f;
+    }
+
+    af::array aT = moddims(a, atdims.ndims(), atdims.get());
+    af::array bT = moddims(b, btdims.ndims(), btdims.get());
+
+    vector<af::array> out(tests.size());
+    if(isBVector) {
+        out[0] = af::matmul(aT, b,    AF_MAT_NONE,    AF_MAT_NONE);
+        out[1] = af::matmul(bT, a,   AF_MAT_NONE,    AF_MAT_NONE);
+        out[2] = af::matmul(b, a,    AF_MAT_TRANS,       AF_MAT_NONE);
+        out[3] = af::matmul(bT, aT,   AF_MAT_NONE,    AF_MAT_TRANS);
+        out[4] = af::matmul(b, aT,    AF_MAT_TRANS,       AF_MAT_TRANS);
+    }
+    else {
+        out[0] = af::matmul(a, b, AF_MAT_NONE,   AF_MAT_NONE);
+        out[1] = af::matmul(a, bT, AF_MAT_NONE,   AF_MAT_TRANS);
+        out[2] = af::matmul(a, bT, AF_MAT_TRANS,      AF_MAT_NONE);
+        out[3] = af::matmul(aT, bT, AF_MAT_TRANS,      AF_MAT_TRANS);
+    }
+
+    for(size_t i = 0; i < tests.size(); i++) {
+        dim_t elems = out[i].elements();
+        vector<T> h_out(elems);
+        out[i].host((void*)&h_out.front());
+
+        if (false == equal(h_out.begin(), h_out.end(), tests[i].begin())) {
+
+            std::cout << "Failed test " << i << "\nCalculated: " << std::endl;
+            std::copy(h_out.begin(), h_out.end(), std::ostream_iterator<T>(std::cout, ", "));
+            std::cout << "Expected: " << std::endl;
+            std::copy(tests[i].begin(), tests[i].end(), std::ostream_iterator<T>(std::cout, ", "));
+            FAIL();
+        }
+    }
+}
+
+#define TEST_FOR_TYPE(TypeName)                                   \
+    tests.emplace_back(cppMatMulCheck<TypeName, false>,                             \
+            nextTargetDeviceId()%numDevices, TEST_DIR "/blas/Basic.test");          \
+    tests.emplace_back(cppMatMulCheck<TypeName, false>,                             \
+            nextTargetDeviceId()%numDevices, TEST_DIR "/blas/NonSquare.test");      \
+    tests.emplace_back(cppMatMulCheck<TypeName, true>,                              \
+            nextTargetDeviceId()%numDevices, TEST_DIR "/blas/SquareVector.test");   \
+    tests.emplace_back(cppMatMulCheck<TypeName, true>,                              \
+            nextTargetDeviceId()%numDevices, TEST_DIR "/blas/RectangleVector.test");
+
+TEST(Threading, BLAS)
+{
+    vector<std::thread> tests;
+
+    int numDevices = 1;
+    ASSERT_EQ(AF_SUCCESS, af_get_device_count(&numDevices));
+
+    TEST_FOR_TYPE(      float);
+    TEST_FOR_TYPE( af::cfloat);
+    TEST_FOR_TYPE(     double);
+    TEST_FOR_TYPE(af::cdouble);
+
+    for (size_t testId=0; testId<tests.size(); ++testId)
+    {
+        if (tests[testId].joinable()) {
+            tests[testId].join();
+        }
+    }
+}

From 8c56c58eba7bb939d9ead3485607fe1673533c18 Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Mon, 3 Apr 2017 17:55:14 +0530
Subject: [PATCH 57/82] Add thread_local qualifier for cu{blas,solver,sparse}
 handles

---
 src/backend/cuda/platform.cpp | 42 +++++++++++++----------------------
 src/backend/cuda/platform.hpp | 14 ------------
 2 files changed, 15 insertions(+), 41 deletions(-)

diff --git a/src/backend/cuda/platform.cpp b/src/backend/cuda/platform.cpp
index 078c1983d4..baaa1ee3eb 100644
--- a/src/backend/cuda/platform.cpp
+++ b/src/backend/cuda/platform.cpp
@@ -436,56 +436,44 @@ PlanCache& fftManager()
 
 BlasHandle blasHandle()
 {
-    static std::once_flag initFlags[DeviceManager::MAX_DEVICES];
+    thread_local static std::unique_ptr<cublasHandle> cublasHandles[DeviceManager::MAX_DEVICES];
+    thread_local static std::once_flag initFlags[DeviceManager::MAX_DEVICES];
 
     int id = cuda::getActiveDeviceId();
 
-    DeviceManager& inst = DeviceManager::getInstance();
+    std::call_once(initFlags[id], [&]{ cublasHandles[id].reset(new cublasHandle()); });
 
-    std::call_once(initFlags[id], [&]{ inst.cublasHandles[id].reset(new cublasHandle()); });
+    CUBLAS_CHECK(cublasSetStream(cublasHandles[id].get()->get(), cuda::getStream(id)));
 
-    return inst.cublasHandles[id].get()->get();
+    return cublasHandles[id].get()->get();
 }
 
 SolveHandle solverDnHandle()
 {
-    static std::once_flag initFlags[DeviceManager::MAX_DEVICES];
+    thread_local static std::unique_ptr<cusolverDnHandle> cusolverHandles[DeviceManager::MAX_DEVICES];
+    thread_local static std::once_flag initFlags[DeviceManager::MAX_DEVICES];
 
     int id = cuda::getActiveDeviceId();
 
-    DeviceManager& inst = DeviceManager::getInstance();
+    std::call_once(initFlags[id], [&]{ cusolverHandles[id].reset(new cusolverDnHandle()); });
 
-    std::call_once(initFlags[id], [&]{ inst.cusolverHandles[id].reset(new cusolverDnHandle()); });
+    CUSOLVER_CHECK(cusolverDnSetStream(cusolverHandles[id].get()->get(), cuda::getStream(id)));
 
-    // FIXME
-    // This is not an ideal case. It's just a hack.
-    // The correct way to do is to use
-    // CUSOLVER_CHECK(cusolverDnSetStream(cuda::getStream(cuda::getActiveDeviceId())))
-    // in the class constructor.
-    // However, this is causing a lot of the cusolver functions to fail.
-    // The only way to fix them is to use cudaDeviceSynchronize() and cudaStreamSynchronize()
-    // all over the place, but even then some calls like getrs in solve_lu
-    // continue to fail on any stream other than 0.
-    //
-    // cuSolver Streams patch:
-    // https://gist.github.com/shehzan10/414c3d04a40e7c4a03ed3c2e1b9072e7
-    //
-    CUDA_CHECK(cudaStreamSynchronize(cuda::getStream(id)));
-
-    return inst.cusolverHandles[id].get()->get();
+    return cusolverHandles[id].get()->get();
 }
 
 SparseHandle sparseHandle()
 {
-    static std::once_flag initFlags[DeviceManager::MAX_DEVICES];
+    thread_local static std::unique_ptr<cusparseHandle> cusparseHandles[DeviceManager::MAX_DEVICES];
+    thread_local static std::once_flag initFlags[DeviceManager::MAX_DEVICES];
 
     int id = cuda::getActiveDeviceId();
 
-    DeviceManager& inst = DeviceManager::getInstance();
+    std::call_once(initFlags[id], [&]{ cusparseHandles[id].reset(new cusparseHandle()); });
 
-    std::call_once(initFlags[id], [&]{ inst.cusparseHandles[id].reset(new cusparseHandle()); });
+    CUSPARSE_CHECK(cusparseSetStream(cusparseHandles[id].get()->get(), cuda::getStream(id)));
 
-    return inst.cusparseHandles[id].get()->get();
+    return cusparseHandles[id].get()->get();
 }
 
 DeviceManager::DeviceManager()
diff --git a/src/backend/cuda/platform.hpp b/src/backend/cuda/platform.hpp
index 152dfc6a14..b9ac3df401 100644
--- a/src/backend/cuda/platform.hpp
+++ b/src/backend/cuda/platform.hpp
@@ -111,14 +111,6 @@ class DeviceManager
         friend GraphicsResourceManager& interopManager();
 #endif
 
-        friend PlanCache& fftManager();
-
-        friend BlasHandle blasHandle();
-
-        friend SolveHandle solverDnHandle();
-
-        friend SparseHandle sparseHandle();
-
         friend std::string getDeviceInfo(int device);
 
         friend std::string getPlatformInfo();
@@ -170,11 +162,5 @@ class DeviceManager
 #if defined(WITH_GRAPHICS)
         std::unique_ptr<GraphicsResourceManager> gfxManagers[MAX_DEVICES];
 #endif
-
-        std::unique_ptr<cublasHandle> cublasHandles[MAX_DEVICES];
-
-        std::unique_ptr<cusolverDnHandle> cusolverHandles[MAX_DEVICES];
-
-        std::unique_ptr<cusparseHandle> cusparseHandles[MAX_DEVICES];
 };
 }

From 895267f260e7dae05cd307be5883e65a993f5eac Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Mon, 3 Apr 2017 17:56:48 +0530
Subject: [PATCH 58/82] Add Solve Unit tests for multi-threaded scenarios

* Moved common solver unit test code to test/solve_common.hpp
  which is used by both solve_dense.cpp and threading.cpp files.
* Disabled blas, solve and fft tests in threading.cpp file
  temporarily for OpenCL backend. They will be re-enabled
  once the issues are resolved.
---
 src/backend/cuda/cublas.cpp   |   1 -
 src/backend/cuda/cusparse.cpp |   1 -
 src/backend/cuda/platform.cpp |  15 +++-
 src/backend/opencl/blas.cpp   |   1 -
 test/solve_common.hpp         | 142 ++++++++++++++++++++++++++++++++++
 test/solve_dense.cpp          | 122 +----------------------------
 test/threading.cpp            |  65 +++++++++++++---
 7 files changed, 213 insertions(+), 134 deletions(-)
 create mode 100644 test/solve_common.hpp

diff --git a/src/backend/cuda/cublas.cpp b/src/backend/cuda/cublas.cpp
index 515a6e4bd0..26d6211c39 100644
--- a/src/backend/cuda/cublas.cpp
+++ b/src/backend/cuda/cublas.cpp
@@ -33,6 +33,5 @@ const char *errorString(cublasStatus_t err)
 void cublasHandle::createHandle(BlasHandle* handle)
 {
     CUBLAS_CHECK(cublasCreate(handle));
-    CUBLAS_CHECK(cublasSetStream(*handle, cuda::getActiveStream()));
 }
 }
diff --git a/src/backend/cuda/cusparse.cpp b/src/backend/cuda/cusparse.cpp
index 526c1c50a3..1e6776fff1 100644
--- a/src/backend/cuda/cusparse.cpp
+++ b/src/backend/cuda/cusparse.cpp
@@ -34,6 +34,5 @@ const char *errorString(cusparseStatus_t err)
 void cusparseHandle::createHandle(SparseHandle* handle)
 {
     CUSPARSE_CHECK(cusparseCreate(handle));
-    CUSPARSE_CHECK(cusparseSetStream(*handle, cuda::getActiveStream()));
 }
 }
diff --git a/src/backend/cuda/platform.cpp b/src/backend/cuda/platform.cpp
index baaa1ee3eb..89cc827758 100644
--- a/src/backend/cuda/platform.cpp
+++ b/src/backend/cuda/platform.cpp
@@ -457,7 +457,20 @@ SolveHandle solverDnHandle()
 
     std::call_once(initFlags[id], [&]{ cusolverHandles[id].reset(new cusolverDnHandle()); });
 
-    CUSOLVER_CHECK(cusolverDnSetStream(cusolverHandles[id].get()->get(), cuda::getStream(id)));
+    //FIXME
+    // This is not an ideal case. It's just a hack.
+    // The correct way to do is to use
+    // CUSOLVER_CHECK(cusolverDnSetStream(cuda::getStream(cuda::getActiveDeviceId())))
+    // in the class constructor.
+    // However, this is causing a lot of the cusolver functions to fail.
+    // The only way to fix them is to use cudaDeviceSynchronize() and
+    //     cudaStreamSynchronize()
+    // all over the place, but even then some calls like getrs in solve_lu
+    // continue to fail on any stream other than 0.
+    //
+    // cuSolver Streams patch:
+    // https://gist.github.com/shehzan10/414c3d04a40e7c4a03ed3c2e1b9072e7
+    CUDA_CHECK(cudaStreamSynchronize(cuda::getStream(id)));
 
     return cusolverHandles[id].get()->get();
 }
diff --git a/src/backend/opencl/blas.cpp b/src/backend/opencl/blas.cpp
index 113a6a6e66..420ee1f332 100644
--- a/src/backend/opencl/blas.cpp
+++ b/src/backend/opencl/blas.cpp
@@ -174,7 +174,6 @@ Array<T> matmul(const Array<T> &lhs, const Array<T> &rhs,
                 (*out.get())(),   out.getOffset(),  out.dims()[0],
                 1, &getQueue()(), 0, nullptr, &event())
             );
-
     }
 
     return out;
diff --git a/test/solve_common.hpp b/test/solve_common.hpp
new file mode 100644
index 0000000000..e5f5520f88
--- /dev/null
+++ b/test/solve_common.hpp
@@ -0,0 +1,142 @@
+/*******************************************************
+ * Copyright (c) 2014, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <gtest/gtest.h>
+#include <arrayfire.h>
+#include <af/dim4.hpp>
+#include <af/defines.h>
+#include <af/traits.hpp>
+#include <vector>
+#include <iostream>
+#include <complex>
+#include <string>
+#include <testHelpers.hpp>
+
+using std::vector;
+using std::string;
+using std::cout;
+using std::endl;
+using std::abs;
+using af::cfloat;
+using af::cdouble;
+
+///////////////////////////////// CPP ////////////////////////////////////
+//
+
+template<typename T>
+void solveTester(const int m, const int n, const int k, double eps, int targetDevice=-1)
+{
+    if (targetDevice>=0)
+        af::setDevice(targetDevice);
+
+    af::deviceGC();
+
+    if (noDoubleTests<T>()) return;
+    if (noLAPACKTests()) return;
+
+#if 1
+    af::array A  = cpu_randu<T>(af::dim4(m, n));
+    af::array X0 = cpu_randu<T>(af::dim4(n, k));
+#else
+    af::array A  = af::randu(m, n, (af::dtype)af::dtype_traits<T>::af_type);
+    af::array X0 = af::randu(n, k, (af::dtype)af::dtype_traits<T>::af_type);
+#endif
+    af::array B0 = af::matmul(A, X0);
+
+    //! [ex_solve]
+    af::array X1 = af::solve(A, B0);
+    //! [ex_solve]
+
+    //! [ex_solve_recon]
+    af::array B1 = af::matmul(A, X1);
+    //! [ex_solve_recon]
+
+    ASSERT_NEAR(0, af::sum<double>(af::abs(real(B0 - B1))) / (m * k), eps);
+    ASSERT_NEAR(0, af::sum<double>(af::abs(imag(B0 - B1))) / (m * k), eps);
+}
+
+template<typename T>
+void solveLUTester(const int n, const int k, double eps, int targetDevice=-1)
+{
+    if (targetDevice>=0)
+        af::setDevice(targetDevice);
+
+    af::deviceGC();
+
+    if (noDoubleTests<T>()) return;
+    if (noLAPACKTests()) return;
+
+#if 1
+    af::array A  = cpu_randu<T>(af::dim4(n, n));
+    af::array X0 = cpu_randu<T>(af::dim4(n, k));
+#else
+    af::array A  = af::randu(n, n, (af::dtype)af::dtype_traits<T>::af_type);
+    af::array X0 = af::randu(n, k, (af::dtype)af::dtype_traits<T>::af_type);
+#endif
+    af::array B0 = af::matmul(A, X0);
+
+    //! [ex_solve_lu]
+    af::array A_lu, pivot;
+    af::lu(A_lu, pivot, A);
+    af::array X1 = af::solveLU(A_lu, pivot, B0);
+    //! [ex_solve_lu]
+
+    af::array B1 = af::matmul(A, X1);
+
+    ASSERT_NEAR(0, af::sum<double>(af::abs(real(B0 - B1))) / (n * k), eps);
+    ASSERT_NEAR(0, af::sum<double>(af::abs(imag(B0 - B1))) / (n * k), eps);
+}
+
+template<typename T>
+void solveTriangleTester(const int n, const int k, bool is_upper, double eps, int targetDevice=-1)
+{
+    if (targetDevice>=0)
+        af::setDevice(targetDevice);
+
+    af::deviceGC();
+
+    if (noDoubleTests<T>()) return;
+    if (noLAPACKTests()) return;
+
+#if 1
+    af::array A  = cpu_randu<T>(af::dim4(n, n));
+    af::array X0 = cpu_randu<T>(af::dim4(n, k));
+#else
+    af::array A  = af::randu(n, n, (af::dtype)af::dtype_traits<T>::af_type);
+    af::array X0 = af::randu(n, k, (af::dtype)af::dtype_traits<T>::af_type);
+#endif
+
+    af::array L, U, pivot;
+    af::lu(L, U, pivot, A);
+
+    af::array AT = is_upper ? U : L;
+    af::array B0 = af::matmul(AT, X0);
+    af::array X1;
+
+    if (is_upper) {
+        //! [ex_solve_upper]
+        af::array X = af::solve(AT, B0, AF_MAT_UPPER);
+        //! [ex_solve_upper]
+
+        X1 = X;
+    } else {
+        //! [ex_solve_lower]
+        af::array X = af::solve(AT, B0, AF_MAT_LOWER);
+        //! [ex_solve_lower]
+
+        X1 = X;
+    }
+
+    af::array B1 = af::matmul(AT, X1);
+
+    ASSERT_NEAR(0, af::sum<double>(af::abs(real(B0 - B1))) / (n * k), eps);
+    ASSERT_NEAR(0, af::sum<double>(af::abs(imag(B0 - B1))) / (n * k), eps);
+}
diff --git a/test/solve_dense.cpp b/test/solve_dense.cpp
index 171551e8cf..641da961ee 100644
--- a/test/solve_dense.cpp
+++ b/test/solve_dense.cpp
@@ -8,127 +8,7 @@
  ********************************************************/
 
 #include <gtest/gtest.h>
-#include <arrayfire.h>
-#include <af/dim4.hpp>
-#include <af/defines.h>
-#include <af/traits.hpp>
-#include <vector>
-#include <iostream>
-#include <complex>
-#include <string>
-#include <testHelpers.hpp>
-
-using std::vector;
-using std::string;
-using std::cout;
-using std::endl;
-using std::abs;
-using af::cfloat;
-using af::cdouble;
-
-///////////////////////////////// CPP ////////////////////////////////////
-//
-
-template<typename T>
-void solveTester(const int m, const int n, const int k, double eps)
-{
-    af::deviceGC();
-
-    if (noDoubleTests<T>()) return;
-    if (noLAPACKTests()) return;
-
-#if 1
-    af::array A  = cpu_randu<T>(af::dim4(m, n));
-    af::array X0 = cpu_randu<T>(af::dim4(n, k));
-#else
-    af::array A  = af::randu(m, n, (af::dtype)af::dtype_traits<T>::af_type);
-    af::array X0 = af::randu(n, k, (af::dtype)af::dtype_traits<T>::af_type);
-#endif
-    af::array B0 = af::matmul(A, X0);
-
-    //! [ex_solve]
-    af::array X1 = af::solve(A, B0);
-    //! [ex_solve]
-
-    //! [ex_solve_recon]
-    af::array B1 = af::matmul(A, X1);
-    //! [ex_solve_recon]
-
-    ASSERT_NEAR(0, af::sum<double>(af::abs(real(B0 - B1))) / (m * k), eps);
-    ASSERT_NEAR(0, af::sum<double>(af::abs(imag(B0 - B1))) / (m * k), eps);
-}
-
-template<typename T>
-void solveLUTester(const int n, const int k, double eps)
-{
-    af::deviceGC();
-
-    if (noDoubleTests<T>()) return;
-    if (noLAPACKTests()) return;
-
-#if 1
-    af::array A  = cpu_randu<T>(af::dim4(n, n));
-    af::array X0 = cpu_randu<T>(af::dim4(n, k));
-#else
-    af::array A  = af::randu(n, n, (af::dtype)af::dtype_traits<T>::af_type);
-    af::array X0 = af::randu(n, k, (af::dtype)af::dtype_traits<T>::af_type);
-#endif
-    af::array B0 = af::matmul(A, X0);
-
-    //! [ex_solve_lu]
-    af::array A_lu, pivot;
-    af::lu(A_lu, pivot, A);
-    af::array X1 = af::solveLU(A_lu, pivot, B0);
-    //! [ex_solve_lu]
-
-    af::array B1 = af::matmul(A, X1);
-
-    ASSERT_NEAR(0, af::sum<double>(af::abs(real(B0 - B1))) / (n * k), eps);
-    ASSERT_NEAR(0, af::sum<double>(af::abs(imag(B0 - B1))) / (n * k), eps);
-}
-
-template<typename T>
-void solveTriangleTester(const int n, const int k, bool is_upper, double eps)
-{
-    af::deviceGC();
-
-    if (noDoubleTests<T>()) return;
-    if (noLAPACKTests()) return;
-
-#if 1
-    af::array A  = cpu_randu<T>(af::dim4(n, n));
-    af::array X0 = cpu_randu<T>(af::dim4(n, k));
-#else
-    af::array A  = af::randu(n, n, (af::dtype)af::dtype_traits<T>::af_type);
-    af::array X0 = af::randu(n, k, (af::dtype)af::dtype_traits<T>::af_type);
-#endif
-
-    af::array L, U, pivot;
-    af::lu(L, U, pivot, A);
-
-    af::array AT = is_upper ? U : L;
-    af::array B0 = af::matmul(AT, X0);
-    af::array X1;
-
-    if (is_upper) {
-        //! [ex_solve_upper]
-        af::array X = af::solve(AT, B0, AF_MAT_UPPER);
-        //! [ex_solve_upper]
-
-        X1 = X;
-    } else {
-        //! [ex_solve_lower]
-        af::array X = af::solve(AT, B0, AF_MAT_LOWER);
-        //! [ex_solve_lower]
-
-        X1 = X;
-    }
-
-    af::array B1 = af::matmul(AT, X1);
-
-    ASSERT_NEAR(0, af::sum<double>(af::abs(real(B0 - B1))) / (n * k), eps);
-    ASSERT_NEAR(0, af::sum<double>(af::abs(imag(B0 - B1))) / (n * k), eps);
-}
+#include "solve_common.hpp"
 
 #define SOLVE_LU_TESTS(T, eps)                          \
     TEST(SOLVE_LU, T##Reg)                              \
diff --git a/test/threading.cpp b/test/threading.cpp
index 16b0314ef4..5ee0396d36 100644
--- a/test/threading.cpp
+++ b/test/threading.cpp
@@ -10,10 +10,13 @@
 #include <cstddef>
 #include <gtest/gtest.h>
 #include <arrayfire.h>
-#include <testHelpers.hpp>
+#include <af/traits.hpp>
 #include <thread>
+#include <complex>
 #include <chrono>
+#include <thread>
 #include <iterator>
+#include <solve_common.hpp>
 
 using namespace af;
 
@@ -28,6 +31,12 @@ static const unsigned ITERATION_COUNT = 10;
 static const unsigned ITERATION_COUNT = 1000;
 #endif
 
+int nextTargetDeviceId()
+{
+    static int nextId = 0;
+    return nextId++;
+}
+
 void morphTest(const array input, const array mask, const bool isDilation,
                const array gold, int targetDevice)
 {
@@ -197,12 +206,6 @@ TEST(Threading, SimultaneousRead)
             tests[t].join();
 }
 
-int nextTargetDeviceId()
-{
-    static int nextId = 0;
-    return nextId++;
-}
-
 static void cleanSlate()
 {
     const size_t step_bytes = 1024;
@@ -336,6 +339,7 @@ TEST(Threading, MemoryManagement_JIT_Node)
     ASSERT_EQ(  lock_bytes,      0u);
 }
 
+#if !defined(AF_OPENCL)
 template<typename inType, typename outType, bool isInverse>
 void fftTest(int targetDevice, string pTestFile, dim_t pad0=0, dim_t pad1=0, dim_t pad2=0)
 {
@@ -413,7 +417,6 @@ void fftTest(int targetDevice, string pTestFile, dim_t pad0=0, dim_t pad1=0, dim
         tests.emplace_back(fftTest<in_t, out_t, is_inverse>, targetDevice, file, p0, p1, 0);\
     }
 
-#if !defined(AF_OPENCL)
 /// OpenCL backend tests seem to be failing even when
 /// each thead has it's own plan cache(thread_local).
 /// The issue seems to present itself randomly in the form of crashes,
@@ -479,7 +482,6 @@ TEST(Threading, FFT)
         }
     }
 }
-#endif
 
 template<typename T, bool isBVector>
 void cppMatMulCheck(int targetDevice, string TestFile)
@@ -574,3 +576,48 @@ TEST(Threading, BLAS)
         }
     }
 }
+
+#define SOLVE_LU_TESTS(T, eps)                                                                          \
+    tests.emplace_back(solveLUTester<T>, 1000, 100, eps, nextTargetDeviceId()%numDevices);              \
+    tests.emplace_back(solveLUTester<T>, 2048, 512, eps, nextTargetDeviceId()%numDevices);              \
+    std::this_thread::sleep_for(std::chrono::seconds(2));   \
+    tests.emplace_back(solveTriangleTester<T>, 1000, 100, true, eps, nextTargetDeviceId()%numDevices);  \
+    tests.emplace_back(solveTriangleTester<T>, 2048, 512, true, eps, nextTargetDeviceId()%numDevices);  \
+    std::this_thread::sleep_for(std::chrono::seconds(2));   \
+    tests.emplace_back(solveTriangleTester<T>, 1000, 100, false, eps, nextTargetDeviceId()%numDevices); \
+    tests.emplace_back(solveTriangleTester<T>, 2048, 512, false, eps, nextTargetDeviceId()%numDevices); \
+    std::this_thread::sleep_for(std::chrono::seconds(2));   \
+    tests.emplace_back(solveTester<T>, 1000, 1000, 100, eps, nextTargetDeviceId()%numDevices);          \
+    tests.emplace_back(solveTester<T>, 2048, 2048, 512, eps, nextTargetDeviceId()%numDevices);          \
+    std::this_thread::sleep_for(std::chrono::seconds(2));   \
+    tests.emplace_back(solveTester<T>, 800, 1000, 200, eps, nextTargetDeviceId()%numDevices);           \
+    tests.emplace_back(solveTester<T>, 1536, 2048, 400, eps, nextTargetDeviceId()%numDevices);          \
+    std::this_thread::sleep_for(std::chrono::seconds(2));   \
+    tests.emplace_back(solveTester<T>, 800, 600, 64, eps, nextTargetDeviceId()%numDevices);             \
+    tests.emplace_back(solveTester<T>, 1536, 1024, 1, eps, nextTargetDeviceId()%numDevices);
+
+// Added 2s sleep for every two test threads to make sure
+// we are not running out of memory.
+TEST(Threading, SolveDense)
+{
+    vector<std::thread> tests;
+
+    int numDevices = 1;
+    ASSERT_EQ(AF_SUCCESS, af_get_device_count(&numDevices));
+
+    SOLVE_LU_TESTS(float, 0.01);
+    SOLVE_LU_TESTS(double, 1E-5);
+    SOLVE_LU_TESTS(cfloat, 0.01);
+    SOLVE_LU_TESTS(cdouble, 1E-5);
+
+    for (size_t testId=0; testId<tests.size(); ++testId)
+    {
+        if (tests[testId].joinable()) {
+            tests[testId].join();
+        }
+    }
+}
+
+#undef SOLVE_LU_TESTS
+
+#endif

From 38662260ff45ee9aa4eaa49adb86f59c0c691ebb Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Sat, 8 Apr 2017 20:32:34 +0530
Subject: [PATCH 59/82] Sparse multi-threaded tests

* OpenCL Backend disabled
---
 test/solve_common.hpp  |   3 -
 test/solve_dense.cpp   |   1 +
 test/sparse.cpp        | 212 +--------------------------------------
 test/sparse_common.hpp | 221 +++++++++++++++++++++++++++++++++++++++++
 test/threading.cpp     |  36 +++++++
 5 files changed, 259 insertions(+), 214 deletions(-)
 create mode 100644 test/sparse_common.hpp

diff --git a/test/solve_common.hpp b/test/solve_common.hpp
index e5f5520f88..3149a7b3f3 100644
--- a/test/solve_common.hpp
+++ b/test/solve_common.hpp
@@ -8,8 +8,6 @@
  ********************************************************/
 
 #pragma once
-
-#include <gtest/gtest.h>
 #include <arrayfire.h>
 #include <af/dim4.hpp>
 #include <af/defines.h>
@@ -18,7 +16,6 @@
 #include <iostream>
 #include <complex>
 #include <string>
-#include <testHelpers.hpp>
 
 using std::vector;
 using std::string;
diff --git a/test/solve_dense.cpp b/test/solve_dense.cpp
index 641da961ee..641b05c7e3 100644
--- a/test/solve_dense.cpp
+++ b/test/solve_dense.cpp
@@ -8,6 +8,7 @@
  ********************************************************/
 
 #include <gtest/gtest.h>
+#include <testHelpers.hpp>
 #include "solve_common.hpp"
 
 #define SOLVE_LU_TESTS(T, eps)                          \
diff --git a/test/sparse.cpp b/test/sparse.cpp
index fc87b95a4d..12bb58be4e 100644
--- a/test/sparse.cpp
+++ b/test/sparse.cpp
@@ -8,149 +8,8 @@
  ********************************************************/
 
 #include <gtest/gtest.h>
-#include <arrayfire.h>
-#include <af/dim4.hpp>
-#include <af/defines.h>
-#include <af/traits.hpp>
-#include <vector>
-#include <iostream>
-#include <complex>
-#include <string>
 #include <testHelpers.hpp>
-
-using std::vector;
-using std::string;
-using std::cout;
-using std::endl;
-using std::abs;
-using af::cfloat;
-using af::cdouble;
-
-///////////////////////////////// CPP ////////////////////////////////////
-//
-
-template<typename T>
-af::array makeSparse(af::array A, int factor)
-{
-    A = floor(A * 1000);
-    A = A * ((A % factor) == 0) / 1000;
-    return A;
-}
-
-template<>
-af::array makeSparse<cfloat>(af::array A, int factor)
-{
-    af::array r = real(A);
-    r = floor(r * 1000);
-    r = r * ((r % factor) == 0) / 1000;
-
-    af::array i = r / 2;
-
-    A = af::complex(r, i);
-    return A;
-}
-
-template<>
-af::array makeSparse<cdouble>(af::array A, int factor)
-{
-    af::array r = real(A);
-    r = floor(r * 1000);
-    r = r * ((r % factor) == 0) / 1000;
-
-    af::array i = r / 2;
-
-    A = af::complex(r, i);
-    return A;
-}
-
-double calc_norm(af::array lhs, af::array rhs)
-{
-    return af::max<double>(af::abs(lhs - rhs) / (af::abs(lhs) + af::abs(rhs) + 1E-5));
-}
-
-template<typename T>
-void sparseTester(const int m, const int n, const int k, int factor, double eps)
-{
-    af::deviceGC();
-
-    if (noDoubleTests<T>()) return;
-
-#if 1
-    af::array A = cpu_randu<T>(af::dim4(m, n));
-    af::array B = cpu_randu<T>(af::dim4(n, k));
-#else
-    af::array A = af::randu(m, n, (af::dtype)af::dtype_traits<T>::af_type);
-    af::array B = af::randu(n, k, (af::dtype)af::dtype_traits<T>::af_type);
-#endif
-
-    A = makeSparse<T>(A, factor);
-
-    // Result of GEMM
-    af::array dRes1 = matmul(A, B);
-
-    // Create Sparse Array From Dense
-    af::array sA = af::sparse(A, AF_STORAGE_CSR);
-
-    // Sparse Matmul
-    af::array sRes1 = matmul(sA, B);
-
-    // Verify Results
-    ASSERT_NEAR(0, calc_norm(real(dRes1), real(sRes1)), eps);
-    ASSERT_NEAR(0, calc_norm(imag(dRes1), imag(sRes1)), eps);
-}
-
-template<typename T>
-void sparseTransposeTester(const int m, const int n, const int k, int factor, double eps)
-{
-    af::deviceGC();
-
-    if (noDoubleTests<T>()) return;
-
-#if 1
-    af::array A = cpu_randu<T>(af::dim4(m, n));
-    af::array B = cpu_randu<T>(af::dim4(m, k));
-#else
-    af::array A = af::randu(m, n, (af::dtype)af::dtype_traits<T>::af_type);
-    af::array B = af::randu(m, k, (af::dtype)af::dtype_traits<T>::af_type);
-#endif
-
-    A = makeSparse<T>(A, factor);
-
-    // Result of GEMM
-    af::array dRes2 = matmul(A, B, AF_MAT_TRANS, AF_MAT_NONE);
-    af::array dRes3 = matmul(A, B, AF_MAT_CTRANS, AF_MAT_NONE);
-
-    // Create Sparse Array From Dense
-    af::array sA = af::sparse(A, AF_STORAGE_CSR);
-
-    // Sparse Matmul
-    af::array sRes2 = matmul(sA, B, AF_MAT_TRANS, AF_MAT_NONE);
-    af::array sRes3 = matmul(sA, B, AF_MAT_CTRANS, AF_MAT_NONE);
-
-    // Verify Results
-    ASSERT_NEAR(0, calc_norm(real(dRes2), real(sRes2)), eps);
-    ASSERT_NEAR(0, calc_norm(imag(dRes2), imag(sRes2)), eps);
-
-    ASSERT_NEAR(0, calc_norm(real(dRes3), real(sRes3)), eps);
-    ASSERT_NEAR(0, calc_norm(imag(dRes3), imag(sRes3)), eps);
-}
-
-template<typename T>
-void convertCSR(const int M, const int N, const float ratio)
-{
-    if (noDoubleTests<T>()) return;
-#if 1
-    af::array a = cpu_randu<T>(af::dim4(M, N));
-#else
-    af::array a = af::randu(M, N);
-#endif
-    a = a * (a > ratio);
-
-    af::array s = af::sparse(a, AF_STORAGE_CSR);
-    af::array aa = af::dense(s);
-
-    ASSERT_EQ(0, af::max<double>(af::abs(a - aa)));
-}
+#include <sparse_common.hpp>
 
 #define SPARSE_TESTS(T, eps)                                \
     TEST(SPARSE, T##Square)                                 \
@@ -197,25 +56,6 @@ SPARSE_TESTS(cdouble, 1E-5)
 
 #undef SPARSE_TESTS
 
-// This test essentially verifies that the sparse structures have the correct
-// dimensions and indices using a very basic test
-template<af_storage stype>
-void createFunction()
-{
-    af::array in = af::sparse(af::identity(3, 3), stype);
-
-    af::array values = sparseGetValues(in);
-    af::array rowIdx = sparseGetRowIdx(in);
-    af::array colIdx = sparseGetColIdx(in);
-    dim_t     nNZ    = sparseGetNNZ(in);
-
-    ASSERT_EQ(nNZ, values.elements());
-
-    ASSERT_EQ(0, af::max<double>(values - af::constant(1, nNZ)));
-    ASSERT_EQ(0, af::max<int   >(rowIdx - af::range(af::dim4(rowIdx.elements()), 0, s32)));
-    ASSERT_EQ(0, af::max<int   >(colIdx - af::range(af::dim4(colIdx.elements()), 0, s32)));
-}
-
 #define CREATE_TESTS(STYPE)                                         \
     TEST(SPARSE_CREATE, STYPE)                                      \
     {                                                               \
@@ -237,56 +77,6 @@ TEST(SPARSE_CREATE, AF_STORAGE_CSC)
     if(out != 0) af_release_array(out);
 }
 
-template<typename Ti, typename To>
-void sparseCastTester(const int m, const int n, int factor)
-{
-    if (noDoubleTests<Ti>()) return;
-    if (noDoubleTests<To>()) return;
-
-    af::array A = cpu_randu<Ti>(af::dim4(m, n));
-
-    A = makeSparse<Ti>(A, factor);
-
-    af::array sTi = af::sparse(A, AF_STORAGE_CSR);
-
-    // Cast
-    af::array sTo = sTi.as((af::dtype)af::dtype_traits<To>::af_type);
-
-    // Verify nnZ
-    dim_t iNNZ = sparseGetNNZ(sTi);
-    dim_t oNNZ = sparseGetNNZ(sTo);
-
-    ASSERT_EQ(iNNZ, oNNZ);
-
-    // Verify Types
-    dim_t iSType = sparseGetStorage(sTi);
-    dim_t oSType = sparseGetStorage(sTo);
-
-    ASSERT_EQ(iSType, oSType);
-
-    // Get the individual arrays and verify equality
-    af::array iValues = sparseGetValues(sTi);
-    af::array iRowIdx = sparseGetRowIdx(sTi);
-    af::array iColIdx = sparseGetColIdx(sTi);
-
-    af::array oValues = sparseGetValues(sTo);
-    af::array oRowIdx = sparseGetRowIdx(sTo);
-    af::array oColIdx = sparseGetColIdx(sTo);
-
-    // Verify values
-    ASSERT_EQ(0, af::max<int>(af::abs(iRowIdx - oRowIdx)));
-    ASSERT_EQ(0, af::max<int>(af::abs(iColIdx - oColIdx)));
-
-    static const double eps = 1e-6;
-    if(iValues.iscomplex() && !oValues.iscomplex()) {
-        ASSERT_NEAR(0, af::max<double>(af::abs(af::abs(iValues) - oValues)), eps);
-    } else if(!iValues.iscomplex() && oValues.iscomplex()) {
-        ASSERT_NEAR(0, af::max<double>(af::abs(iValues - af::abs(oValues))), eps);
-    } else {
-        ASSERT_NEAR(0, af::max<double>(af::abs(iValues - oValues)), eps);
-    }
-}
-
 #define CAST_TESTS_TYPES(Ti, To, SUFFIX, M, N, F)                               \
     TEST(SPARSE_CAST, Ti##_##To##_##SUFFIX)                                     \
     {                                                                           \
diff --git a/test/sparse_common.hpp b/test/sparse_common.hpp
new file mode 100644
index 0000000000..57705d3ca0
--- /dev/null
+++ b/test/sparse_common.hpp
@@ -0,0 +1,221 @@
+/*******************************************************
+ * Copyright (c) 2017, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+#include <arrayfire.h>
+#include <af/dim4.hpp>
+#include <af/defines.h>
+#include <af/traits.hpp>
+#include <vector>
+#include <iostream>
+#include <complex>
+#include <string>
+
+using std::vector;
+using std::string;
+using std::cout;
+using std::endl;
+using std::abs;
+using af::cfloat;
+using af::cdouble;
+
+///////////////////////////////// CPP ////////////////////////////////////
+//
+
+template<typename T>
+af::array makeSparse(af::array A, int factor)
+{
+    A = floor(A * 1000);
+    A = A * ((A % factor) == 0) / 1000;
+    return A;
+}
+
+template<>
+af::array makeSparse<cfloat>(af::array A, int factor)
+{
+    af::array r = real(A);
+    r = floor(r * 1000);
+    r = r * ((r % factor) == 0) / 1000;
+
+    af::array i = r / 2;
+
+    A = af::complex(r, i);
+    return A;
+}
+
+template<>
+af::array makeSparse<cdouble>(af::array A, int factor)
+{
+    af::array r = real(A);
+    r = floor(r * 1000);
+    r = r * ((r % factor) == 0) / 1000;
+
+    af::array i = r / 2;
+
+    A = af::complex(r, i);
+    return A;
+}
+
+double calc_norm(af::array lhs, af::array rhs)
+{
+    return af::max<double>(af::abs(lhs - rhs) / (af::abs(lhs) + af::abs(rhs) + 1E-5));
+}
+
+template<typename T>
+void sparseTester(const int m, const int n, const int k, int factor, double eps)
+{
+    af::deviceGC();
+
+    if (noDoubleTests<T>()) return;
+
+#if 1
+    af::array A = cpu_randu<T>(af::dim4(m, n));
+    af::array B = cpu_randu<T>(af::dim4(n, k));
+#else
+    af::array A = af::randu(m, n, (af::dtype)af::dtype_traits<T>::af_type);
+    af::array B = af::randu(n, k, (af::dtype)af::dtype_traits<T>::af_type);
+#endif
+
+    A = makeSparse<T>(A, factor);
+
+    // Result of GEMM
+    af::array dRes1 = matmul(A, B);
+
+    // Create Sparse Array From Dense
+    af::array sA = af::sparse(A, AF_STORAGE_CSR);
+
+    // Sparse Matmul
+    af::array sRes1 = matmul(sA, B);
+
+    // Verify Results
+    ASSERT_NEAR(0, calc_norm(real(dRes1), real(sRes1)), eps);
+    ASSERT_NEAR(0, calc_norm(imag(dRes1), imag(sRes1)), eps);
+}
+
+template<typename T>
+void sparseTransposeTester(const int m, const int n, const int k, int factor, double eps)
+{
+    af::deviceGC();
+
+    if (noDoubleTests<T>()) return;
+
+#if 1
+    af::array A = cpu_randu<T>(af::dim4(m, n));
+    af::array B = cpu_randu<T>(af::dim4(m, k));
+#else
+    af::array A = af::randu(m, n, (af::dtype)af::dtype_traits<T>::af_type);
+    af::array B = af::randu(m, k, (af::dtype)af::dtype_traits<T>::af_type);
+#endif
+
+    A = makeSparse<T>(A, factor);
+
+    // Result of GEMM
+    af::array dRes2 = matmul(A, B, AF_MAT_TRANS, AF_MAT_NONE);
+    af::array dRes3 = matmul(A, B, AF_MAT_CTRANS, AF_MAT_NONE);
+
+    // Create Sparse Array From Dense
+    af::array sA = af::sparse(A, AF_STORAGE_CSR);
+
+    // Sparse Matmul
+    af::array sRes2 = matmul(sA, B, AF_MAT_TRANS, AF_MAT_NONE);
+    af::array sRes3 = matmul(sA, B, AF_MAT_CTRANS, AF_MAT_NONE);
+
+    // Verify Results
+    ASSERT_NEAR(0, calc_norm(real(dRes2), real(sRes2)), eps);
+    ASSERT_NEAR(0, calc_norm(imag(dRes2), imag(sRes2)), eps);
+
+    ASSERT_NEAR(0, calc_norm(real(dRes3), real(sRes3)), eps);
+    ASSERT_NEAR(0, calc_norm(imag(dRes3), imag(sRes3)), eps);
+}
+
+template<typename T>
+void convertCSR(const int M, const int N, const float ratio)
+{
+    if (noDoubleTests<T>()) return;
+#if 1
+    af::array a = cpu_randu<T>(af::dim4(M, N));
+#else
+    af::array a = af::randu(M, N);
+#endif
+    a = a * (a > ratio);
+
+    af::array s = af::sparse(a, AF_STORAGE_CSR);
+    af::array aa = af::dense(s);
+
+    ASSERT_EQ(0, af::max<double>(af::abs(a - aa)));
+}
+
+// This test essentially verifies that the sparse structures have the correct
+// dimensions and indices using a very basic test
+template<af_storage stype>
+void createFunction()
+{
+    af::array in = af::sparse(af::identity(3, 3), stype);
+
+    af::array values = sparseGetValues(in);
+    af::array rowIdx = sparseGetRowIdx(in);
+    af::array colIdx = sparseGetColIdx(in);
+    dim_t     nNZ    = sparseGetNNZ(in);
+
+    ASSERT_EQ(nNZ, values.elements());
+
+    ASSERT_EQ(0, af::max<double>(values - af::constant(1, nNZ)));
+    ASSERT_EQ(0, af::max<int   >(rowIdx - af::range(af::dim4(rowIdx.elements()), 0, s32)));
+    ASSERT_EQ(0, af::max<int   >(colIdx - af::range(af::dim4(colIdx.elements()), 0, s32)));
+}
+
+template<typename Ti, typename To>
+void sparseCastTester(const int m, const int n, int factor)
+{
+    if (noDoubleTests<Ti>()) return;
+    if (noDoubleTests<To>()) return;
+
+    af::array A = cpu_randu<Ti>(af::dim4(m, n));
+
+    A = makeSparse<Ti>(A, factor);
+
+    af::array sTi = af::sparse(A, AF_STORAGE_CSR);
+
+    // Cast
+    af::array sTo = sTi.as((af::dtype)af::dtype_traits<To>::af_type);
+
+    // Verify nnZ
+    dim_t iNNZ = sparseGetNNZ(sTi);
+    dim_t oNNZ = sparseGetNNZ(sTo);
+
+    ASSERT_EQ(iNNZ, oNNZ);
+
+    // Verify Types
+    dim_t iSType = sparseGetStorage(sTi);
+    dim_t oSType = sparseGetStorage(sTo);
+
+    ASSERT_EQ(iSType, oSType);
+
+    // Get the individual arrays and verify equality
+    af::array iValues = sparseGetValues(sTi);
+    af::array iRowIdx = sparseGetRowIdx(sTi);
+    af::array iColIdx = sparseGetColIdx(sTi);
+
+    af::array oValues = sparseGetValues(sTo);
+    af::array oRowIdx = sparseGetRowIdx(sTo);
+    af::array oColIdx = sparseGetColIdx(sTo);
+
+    // Verify values
+    ASSERT_EQ(0, af::max<int>(af::abs(iRowIdx - oRowIdx)));
+    ASSERT_EQ(0, af::max<int>(af::abs(iColIdx - oColIdx)));
+
+    static const double eps = 1e-6;
+    if(iValues.iscomplex() && !oValues.iscomplex()) {
+        ASSERT_NEAR(0, af::max<double>(af::abs(af::abs(iValues) - oValues)), eps);
+    } else if(!iValues.iscomplex() && oValues.iscomplex()) {
+        ASSERT_NEAR(0, af::max<double>(af::abs(iValues - af::abs(oValues))), eps);
+    } else {
+        ASSERT_NEAR(0, af::max<double>(af::abs(iValues - oValues)), eps);
+    }
+}
diff --git a/test/threading.cpp b/test/threading.cpp
index 5ee0396d36..d5df9cf0c1 100644
--- a/test/threading.cpp
+++ b/test/threading.cpp
@@ -16,7 +16,9 @@
 #include <chrono>
 #include <thread>
 #include <iterator>
+#include <testHelpers.hpp>
 #include <solve_common.hpp>
+#include <sparse_common.hpp>
 
 using namespace af;
 
@@ -620,4 +622,38 @@ TEST(Threading, SolveDense)
 
 #undef SOLVE_LU_TESTS
 
+#define SPARSE_TESTS(T, eps)                                                    \
+        tests.emplace_back(sparseTester<T>, 1000, 1000, 100, 5, eps);           \
+        tests.emplace_back(sparseTester<T>, 2048, 1024, 512, 3, eps);           \
+        tests.emplace_back(sparseTester<T>, 500, 1000, 250, 1, eps);            \
+        tests.emplace_back(sparseTester<T>, 625, 1331, 1, 2, eps);              \
+        tests.emplace_back(sparseTransposeTester<T>, 625, 1331, 1, 2, eps);     \
+        tests.emplace_back(sparseTransposeTester<T>, 1000, 1000, 100, 5, eps);  \
+        tests.emplace_back(sparseTransposeTester<T>, 2048, 1024, 512, 3, eps);  \
+        tests.emplace_back(sparseTransposeTester<T>, 453, 751, 397, 1, eps);    \
+        tests.emplace_back(convertCSR<T>, 2345, 5678, 0.5);                     \
+        std::this_thread::sleep_for(std::chrono::seconds(5));
+
+// Added 2s sleep for every two test threads to make sure
+// we are not running out of memory.
+TEST(Threading, Sparse)
+{
+    vector<std::thread> tests;
+
+    int numDevices = 1;
+    ASSERT_EQ(AF_SUCCESS, af_get_device_count(&numDevices));
+
+    SPARSE_TESTS(  float, 1E-3);
+    SPARSE_TESTS( double, 1E-5);
+    SPARSE_TESTS( cfloat, 1E-3);
+    SPARSE_TESTS(cdouble, 1E-5);
+
+    for (size_t testId=0; testId<tests.size(); ++testId)
+    {
+        if (tests[testId].joinable()) {
+            tests[testId].join();
+        }
+    }
+}
+
 #endif

From 9809394483aa56c3d4941a0fcc4fc1d6925f5dbb Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Mon, 10 Apr 2017 22:45:55 +0530
Subject: [PATCH 60/82] fix bug in fft cache size modification

---
 src/backend/common/FFTPlanCache.hpp | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/backend/common/FFTPlanCache.hpp b/src/backend/common/FFTPlanCache.hpp
index 3d18f31117..c2d9d7c9bd 100644
--- a/src/backend/common/FFTPlanCache.hpp
+++ b/src/backend/common/FFTPlanCache.hpp
@@ -31,7 +31,13 @@ class FFTPlanCache
     public:
         FFTPlanCache() : mMaxCacheSize(5) {}
 
-        void setMaxCacheSize(size_t size) { mMaxCacheSize = size; }
+        void setMaxCacheSize(size_t size)
+        {
+            mMaxCacheSize = size;
+            while (mCache.size()>mMaxCacheSize)
+                mCache.pop_back();
+        }
+
         size_t getMaxCacheSize() const    { return mMaxCacheSize; }
 
         // iterates through plan cache from front to back

From 8c10b3faa31aaea723d153990888905623709db0 Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Mon, 10 Apr 2017 22:46:30 +0530
Subject: [PATCH 61/82] C++ interface for fft cache size modifier fn

---
 include/af/signal.h | 14 ++++++++++++++
 src/api/cpp/fft.cpp |  6 ++++--
 2 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/include/af/signal.h b/include/af/signal.h
index ffa0c108af..fa04bc8dbb 100644
--- a/include/af/signal.h
+++ b/include/af/signal.h
@@ -654,6 +654,18 @@ AFAPI array medfilt1(const array& in, const dim_t wind_width = 3, const borderTy
 AFAPI array medfilt2(const array& in, const dim_t wind_length = 3, const dim_t wind_width = 3, const borderType edge_pad = AF_PAD_ZERO);
 #endif
 
+#if AF_API_VERSION >= 35
+/**
+   C++ Interface for setting plan cache size
+
+   This function doesn't do anything if called when CPU backend is active. The plans associated with
+   the most recently used array sizes are cached.
+
+   \param[in] cacheSize is the number of plans that shall be cached
+*/
+AFAPI void setFFTPlanCacheSize(size_t cacheSize);
+#endif
+
 }
 #endif
 
@@ -1190,6 +1202,8 @@ AFAPI af_err af_iir(af_array *y, const af_array b, const af_array a, const af_ar
    the most recently used array sizes are cached.
 
    \param[in] cache_size is the number of plans that shall be cached
+
+   \ingroup signal_func_fft
 */
 AFAPI af_err af_set_fft_plan_cache_size(size_t cache_size);
 #endif
diff --git a/src/api/cpp/fft.cpp b/src/api/cpp/fft.cpp
index 1fe4a0921c..e7e96ca195 100644
--- a/src/api/cpp/fft.cpp
+++ b/src/api/cpp/fft.cpp
@@ -14,8 +14,6 @@
 
 namespace af
 {
-
-
 array fftNorm(const array& in, const double norm_factor, const dim_t odim0)
 {
     af_array out = 0;
@@ -281,4 +279,8 @@ FFT_REAL(1)
 FFT_REAL(2)
 FFT_REAL(3)
 
+void setFFTPlanCacheSize(size_t cacheSize)
+{
+    AF_THROW(af_set_fft_plan_cache_size(cacheSize));
+}
 }

From c239b86b1ceed3837e2e03c7940f2e20d2c9da24 Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Wed, 12 Apr 2017 00:53:58 +0530
Subject: [PATCH 62/82] Use global kernel cache for all opencl kernels

Earlier to this change, all the kernel wrapper functions used to
maintain a set of static objects that are compiled only once.
---
 src/backend/opencl/kernel/approx.hpp          | 275 +++++++++---------
 src/backend/opencl/kernel/assign.hpp          |  49 ++--
 src/backend/opencl/kernel/bilateral.hpp       |  66 ++---
 .../opencl/kernel/convolve/conv_common.hpp    |  61 ++--
 src/backend/opencl/kernel/diagonal.hpp        | 153 +++++-----
 src/backend/opencl/kernel/diff.hpp            |  96 +++---
 src/backend/opencl/kernel/exampleFunction.hpp |  83 +++---
 src/backend/opencl/kernel/fftconvolve.hpp     | 251 +++++++++-------
 src/backend/opencl/kernel/gradient.hpp        | 103 +++----
 src/backend/opencl/kernel/harris.hpp          | 127 ++++----
 src/backend/opencl/kernel/histogram.hpp       |  60 ++--
 src/backend/opencl/kernel/homography.hpp      | 171 ++++++-----
 src/backend/opencl/kernel/hsv_rgb.hpp         |  43 ++-
 src/backend/opencl/kernel/identity.hpp        |  68 ++---
 src/backend/opencl/kernel/iir.hpp             |  99 +++----
 src/backend/opencl/kernel/index.hpp           |  41 ++-
 src/backend/opencl/kernel/iota.hpp            |  89 +++---
 src/backend/opencl/kernel/join.hpp            |  93 +++---
 src/backend/opencl/kernel/laset.hpp           |  55 ++--
 src/backend/opencl/kernel/laset_band.hpp      |  49 ++--
 src/backend/opencl/kernel/laswp.hpp           |  56 ++--
 src/backend/opencl/kernel/lookup.hpp          |  65 ++---
 src/backend/opencl/kernel/lu_split.hpp        |  70 ++---
 src/backend/opencl/kernel/match_template.hpp  |  76 +++--
 src/backend/opencl/kernel/meanshift.hpp       |  53 ++--
 src/backend/opencl/kernel/medfilt.hpp         | 132 ++++-----
 src/backend/opencl/kernel/memcopy.hpp         | 242 +++++++--------
 src/backend/opencl/kernel/morph.hpp           | 122 ++++----
 src/backend/opencl/kernel/orb.hpp             |  99 ++++---
 src/backend/opencl/kernel/range.hpp           |  81 +++---
 src/backend/opencl/kernel/regions.hpp         | 147 +++++-----
 src/backend/opencl/kernel/reorder.hpp         |  88 +++---
 src/backend/opencl/kernel/resize.hpp          | 161 +++++-----
 src/backend/opencl/kernel/rotate.hpp          | 232 ++++++++-------
 src/backend/opencl/kernel/select.hpp          | 222 +++++++-------
 src/backend/opencl/kernel/shift.hpp           | 102 +++----
 src/backend/opencl/kernel/sift_nonfree.hpp    | 126 ++++----
 src/backend/opencl/kernel/sobel.hpp           |  60 ++--
 src/backend/opencl/kernel/susan.hpp           | 114 ++++----
 src/backend/opencl/kernel/swapdblk.hpp        |  49 ++--
 src/backend/opencl/kernel/tile.hpp            |  83 +++---
 src/backend/opencl/kernel/transform.hpp       |   6 +-
 src/backend/opencl/kernel/transpose.hpp       |  53 ++--
 .../opencl/kernel/transpose_inplace.hpp       |  54 ++--
 src/backend/opencl/kernel/triangle.hpp        |  64 ++--
 src/backend/opencl/kernel/where.hpp           | 206 ++++++-------
 46 files changed, 2327 insertions(+), 2468 deletions(-)

diff --git a/src/backend/opencl/kernel/approx.hpp b/src/backend/opencl/kernel/approx.hpp
index 85d3ba579c..21a0274384 100644
--- a/src/backend/opencl/kernel/approx.hpp
+++ b/src/backend/opencl/kernel/approx.hpp
@@ -14,8 +14,6 @@
 #include <program.hpp>
 #include <traits.hpp>
 #include <string>
-#include <mutex>
-#include <map>
 #include <dispatch.hpp>
 #include <Param.hpp>
 #include <debug_opencl.hpp>
@@ -34,148 +32,135 @@ using std::string;
 
 namespace opencl
 {
-    namespace kernel
-    {
-        static const int TX = 16;
-        static const int TY = 16;
-
-        static const int THREADS = 256;
-
-        ///////////////////////////////////////////////////////////////////////////
-        // Wrapper functions
-        ///////////////////////////////////////////////////////////////////////////
-        template <typename Ty, typename Tp, int order>
-        void approx1(Param out, const Param in, const Param xpos, const float offGrid,
-                     af_interp_type method)
-        {
-            static std::once_flag compileFlags[DeviceManager::MAX_DEVICES];
-            static std::map<int, Program*>  approxProgs;
-            static std::map<int, Kernel*> approxKernels;
-
-            int device = getActiveDeviceId();
-
-            std::call_once( compileFlags[device], [device] () {
-                ToNumStr<Ty> toNumStr;
-                std::ostringstream options;
-                options << " -D Ty="          << dtype_traits<Ty>::getName()
-                        << " -D Tp="          << dtype_traits<Tp>::getName()
-                        << " -D InterpInTy="  << dtype_traits<Ty>::getName()
-                        << " -D InterpValTy=" << dtype_traits<Ty>::getName()
-                        << " -D InterpPosTy=" << dtype_traits<Tp>::getName()
-                        << " -D ZERO="        << toNumStr(scalar<Ty>(0));
-
-                if((af_dtype) dtype_traits<Ty>::af_type == c32 ||
-                    (af_dtype) dtype_traits<Ty>::af_type == c64) {
-                    options << " -D IS_CPLX=1";
-                } else {
-                    options << " -D IS_CPLX=0";
-                }
-                if (std::is_same<Ty, double>::value ||
-                    std::is_same<Ty, cdouble>::value) {
-                    options << " -D USE_DOUBLE";
-                }
-
-                options << " -D INTERP_ORDER=" << order;
-                addInterpEnumOptions(options);
-
-                Program prog;
-                const char *ker_strs[] = {interp_cl, approx1_cl};
-                const int   ker_lens[] = {interp_cl_len, approx1_cl_len};
-                buildProgram(prog, 2, ker_strs, ker_lens, options.str());
-                approxProgs[device] = new Program(prog);
-
-                approxKernels[device] = new Kernel(*approxProgs[device], "approx1_kernel");
-            });
-
-
-            auto approx1Op = KernelFunctor<Buffer, const KParam, const Buffer, const KParam,
-                                            const Buffer, const KParam, const Ty,
-                                            const int, const int, const int>
-                                  (*approxKernels[device]);
-
-            NDRange local(THREADS, 1, 1);
-            dim_t blocksPerMat = divup(out.info.dims[0], local[0]);
-            NDRange global(blocksPerMat * local[0] * out.info.dims[1],
-                            out.info.dims[2] * out.info.dims[3] * local[0],
-                            1);
-
-            // Passing bools to opencl kernels is not allowed
-            bool batch = !(xpos.info.dims[1] == 1 && xpos.info.dims[2] == 1 &&
-                            xpos.info.dims[3] == 1);
-
-            approx1Op(EnqueueArgs(getQueue(), global, local),
-                      *out.data, out.info, *in.data, in.info,
-                      *xpos.data, xpos.info, scalar<Ty>(offGrid),
-                      blocksPerMat, (int)batch, (int)method);
-
-            CL_DEBUG_FINISH(getQueue());
-        }
-
-        template <typename Ty, typename Tp, int order>
-        void approx2(Param out, const Param in, const Param xpos, const Param ypos,
-                     const float offGrid, af_interp_type method)
-        {
-            static std::once_flag compileFlags[DeviceManager::MAX_DEVICES];
-            static std::map<int, Program*>       approxProgs;
-            static std::map<int, Kernel*>      approxKernels;
-
-            int device = getActiveDeviceId();
-
-            std::call_once( compileFlags[device], [device] () {
-                ToNumStr<Ty> toNumStr;
-                std::ostringstream options;
-                options << " -D Ty="          << dtype_traits<Ty>::getName()
-                        << " -D Tp="          << dtype_traits<Tp>::getName()
-                        << " -D InterpInTy="  << dtype_traits<Ty>::getName()
-                        << " -D InterpValTy=" << dtype_traits<Ty>::getName()
-                        << " -D InterpPosTy=" << dtype_traits<Tp>::getName()
-                        << " -D ZERO="        << toNumStr(scalar<Ty>(0));
-
-                if((af_dtype) dtype_traits<Ty>::af_type == c32 ||
-                    (af_dtype) dtype_traits<Ty>::af_type == c64) {
-                    options << " -D IS_CPLX=1";
-                } else {
-                    options << " -D IS_CPLX=0";
-                }
-                if (std::is_same<Ty, double>::value ||
-                    std::is_same<Ty, cdouble>::value) {
-                    options << " -D USE_DOUBLE";
-                }
-
-                options << " -D INTERP_ORDER=" << order;
-                addInterpEnumOptions(options);
-
-                Program prog;
-                const char *ker_strs[] = {interp_cl, approx2_cl};
-                const int   ker_lens[] = {interp_cl_len, approx2_cl_len};
-                buildProgram(prog, 2, ker_strs, ker_lens, options.str());
-                approxProgs[device] = new Program(prog);
-
-                approxKernels[device] = new Kernel(*approxProgs[device], "approx2_kernel");
-            });
-
-            auto approx2Op = KernelFunctor<Buffer, const KParam, const Buffer, const KParam,
-                                    const Buffer, const KParam, const Buffer, const KParam,
-                                            const Ty, const int, const int, const int, const int>
-                                    (*approxKernels[device]);
-
-            NDRange local(TX, TY, 1);
-            dim_t blocksPerMatX = divup(out.info.dims[0], local[0]);
-            dim_t blocksPerMatY = divup(out.info.dims[1], local[1]);
-            NDRange global(blocksPerMatX * local[0] * out.info.dims[2],
-                            blocksPerMatY * local[1] * out.info.dims[3],
-                            1);
-
-            // Passing bools to opencl kernels is not allowed
-            bool batch = !(xpos.info.dims[2] == 1 && xpos.info.dims[3] == 1);
-
-            approx2Op(EnqueueArgs(getQueue(), global, local),
-                      *out.data, out.info,
-                      *in.data, in.info,
-                      *xpos.data, xpos.info,
-                      *ypos.data, ypos.info,
-                      scalar<Ty>(offGrid), blocksPerMatX, blocksPerMatY, (int)batch, (int)method);
-            CL_DEBUG_FINISH(getQueue());
-        }
+namespace kernel
+{
+static const int TX = 16;
+static const int TY = 16;
+
+static const int THREADS = 256;
+
+template<typename Ty, typename Tp, int order>
+std::string generateOptionsString()
+{
+    ToNumStr<Ty> toNumStr;
+
+    std::ostringstream options;
+    options << " -D Ty="          << dtype_traits<Ty>::getName()
+        << " -D Tp="          << dtype_traits<Tp>::getName()
+        << " -D InterpInTy="  << dtype_traits<Ty>::getName()
+        << " -D InterpValTy=" << dtype_traits<Ty>::getName()
+        << " -D InterpPosTy=" << dtype_traits<Tp>::getName()
+        << " -D ZERO="        << toNumStr(scalar<Ty>(0));
+
+    if((af_dtype) dtype_traits<Ty>::af_type == c32 ||
+            (af_dtype) dtype_traits<Ty>::af_type == c64) {
+        options << " -D IS_CPLX=1";
+    } else {
+        options << " -D IS_CPLX=0";
     }
+    if (std::is_same<Ty, double>::value ||
+            std::is_same<Ty, cdouble>::value) {
+        options << " -D USE_DOUBLE";
+    }
+
+    options << " -D INTERP_ORDER=" << order;
+    addInterpEnumOptions(options);
+
+    return options.str();
+}
+
+///////////////////////////////////////////////////////////////////////////
+// Wrapper functions
+///////////////////////////////////////////////////////////////////////////
+template <typename Ty, typename Tp, int order>
+void approx1(Param out, const Param in, const Param xpos, const float offGrid,
+             af_interp_type method)
+{
+    std::string refName = std::string("approx1_kernel_") +
+        std::string(dtype_traits<Ty>::getName()) +
+        std::string(dtype_traits<Tp>::getName()) +
+        std::to_string(order);
+
+    int device = getActiveDeviceId();
+    kc_entry_t entry = kernelCache(device, refName);
+
+    if (entry.prog==0 && entry.ker==0) {
+        std::string options = generateOptionsString<Ty, Tp, order>();
+
+        const char *ker_strs[] = {interp_cl, approx1_cl};
+        const int   ker_lens[] = {interp_cl_len, approx1_cl_len};
+        Program prog;
+        buildProgram(prog, 2, ker_strs, ker_lens, options);
+        entry.prog = new Program(prog);
+        entry.ker  = new Kernel(*entry.prog, "approx1_kernel");
+
+        addKernelToCache(device, refName, entry);
+    }
+
+    auto approx1Op = KernelFunctor< Buffer, const KParam, const Buffer, const KParam,
+                                    const Buffer, const KParam, const Ty,
+                                    const int, const int, const int >(*entry.ker);
+
+    NDRange local(THREADS, 1, 1);
+    dim_t blocksPerMat = divup(out.info.dims[0], local[0]);
+    NDRange global(blocksPerMat * local[0] * out.info.dims[1],
+                   out.info.dims[2] * out.info.dims[3] * local[0], 1);
+
+    // Passing bools to opencl kernels is not allowed
+    bool batch = !(xpos.info.dims[1] == 1 && xpos.info.dims[2] == 1 && xpos.info.dims[3] == 1);
+
+    approx1Op(EnqueueArgs(getQueue(), global, local),
+              *out.data, out.info, *in.data, in.info,
+              *xpos.data, xpos.info, scalar<Ty>(offGrid),
+              blocksPerMat, (int)batch, (int)method);
+
+    CL_DEBUG_FINISH(getQueue());
+}
+
+template <typename Ty, typename Tp, int order>
+void approx2(Param out, const Param in, const Param xpos, const Param ypos,
+             const float offGrid, af_interp_type method)
+{
+    std::string refName = std::string("approx2_kernel_") +
+        std::string(dtype_traits<Ty>::getName()) +
+        std::string(dtype_traits<Tp>::getName()) +
+        std::to_string(order);
+
+    int device = getActiveDeviceId();
+    kc_entry_t entry = kernelCache(device, refName);
+
+    if (entry.prog==0 && entry.ker==0) {
+        std::string options = generateOptionsString<Ty, Tp, order>();
+
+        const char *ker_strs[] = {interp_cl, approx2_cl};
+        const int   ker_lens[] = {interp_cl_len, approx2_cl_len};
+        Program prog;
+        buildProgram(prog, 2, ker_strs, ker_lens, options);
+        entry.prog = new Program(prog);
+        entry.ker  = new Kernel(*entry.prog, "approx2_kernel");
+
+        addKernelToCache(device, refName, entry);
+    }
+
+    auto approx2Op = KernelFunctor< Buffer, const KParam, const Buffer, const KParam,
+                                    const Buffer, const KParam, const Buffer, const KParam,
+                                    const Ty, const int, const int, const int, const int >(*entry.ker);
+
+    NDRange local(TX, TY, 1);
+    dim_t blocksPerMatX = divup(out.info.dims[0], local[0]);
+    dim_t blocksPerMatY = divup(out.info.dims[1], local[1]);
+    NDRange global(blocksPerMatX * local[0] * out.info.dims[2],
+                   blocksPerMatY * local[1] * out.info.dims[3], 1);
+
+    // Passing bools to opencl kernels is not allowed
+    bool batch = !(xpos.info.dims[2] == 1 && xpos.info.dims[3] == 1);
+
+    approx2Op(EnqueueArgs(getQueue(), global, local),
+              *out.data, out.info, *in.data, in.info,
+              *xpos.data, xpos.info, *ypos.data, ypos.info,
+              scalar<Ty>(offGrid), blocksPerMatX, blocksPerMatY, (int)batch, (int)method);
+
+    CL_DEBUG_FINISH(getQueue());
+}
+}
 }
diff --git a/src/backend/opencl/kernel/assign.hpp b/src/backend/opencl/kernel/assign.hpp
index 079768c4a9..580e7ab582 100644
--- a/src/backend/opencl/kernel/assign.hpp
+++ b/src/backend/opencl/kernel/assign.hpp
@@ -12,8 +12,7 @@
 #include <program.hpp>
 #include <traits.hpp>
 #include <string>
-#include <mutex>
-#include <map>
+#include <cache.hpp>
 #include <dispatch.hpp>
 #include <Param.hpp>
 #include <debug_opencl.hpp>
@@ -28,10 +27,8 @@ using std::string;
 
 namespace opencl
 {
-
 namespace kernel
 {
-
 static const int THREADS_X = 32;
 static const int THREADS_Y =  8;
 
@@ -44,46 +41,42 @@ typedef struct {
 template<typename T>
 void assign(Param out, const Param in, const AssignKernelParam_t& p, Buffer *bPtr[4])
 {
-    static std::once_flag compileFlags[DeviceManager::MAX_DEVICES];
-    static std::map<int, Program*>  agnProgs;
-    static std::map<int, Kernel*> agnKernels;
+    std::string refName = std::string("assignKernel_") + std::string(dtype_traits<T>::getName());
 
     int device = getActiveDeviceId();
+    kc_entry_t entry = kernelCache(device, refName);
 
-    std::call_once( compileFlags[device], [device] () {
-            std::ostringstream options;
-            options << " -D T=" << dtype_traits<T>::getName();
-
-            if (std::is_same<T, double>::value ||
-                std::is_same<T, cdouble>::value) {
+    if (entry.prog==0 && entry.ker==0) {
+        std::ostringstream options;
+        options << " -D T=" << dtype_traits<T>::getName();
+        if (std::is_same<T, double>::value || std::is_same<T, cdouble>::value)
             options << " -D USE_DOUBLE";
-            }
 
-            Program prog;
-            buildProgram(prog, assign_cl, assign_cl_len, options.str());
-            agnProgs[device]   = new Program(prog);
-            agnKernels[device] = new Kernel(*agnProgs[device], "assignKernel");
-            });
+        const char* ker_strs[] = {assign_cl};
+        const int   ker_lens[] = {assign_cl_len};
+        Program prog;
+        buildProgram(prog, 1, ker_strs, ker_lens, options.str());
+        entry.prog = new Program(prog);
+        entry.ker  = new Kernel(*entry.prog, "assignKernel");
+
+        addKernelToCache(device, refName, entry);
+    }
 
     NDRange local(THREADS_X, THREADS_Y);
 
     int blk_x = divup(in.info.dims[0], THREADS_X);
     int blk_y = divup(in.info.dims[1], THREADS_Y);
 
-    NDRange global(blk_x * in.info.dims[2] * THREADS_X,
-            blk_y * in.info.dims[3] * THREADS_Y);
+    NDRange global(blk_x * in.info.dims[2] * THREADS_X, blk_y * in.info.dims[3] * THREADS_Y);
 
-    auto assignOp = KernelFunctor<Buffer, KParam, Buffer, KParam, AssignKernelParam_t,
-          Buffer, Buffer, Buffer, Buffer, int, int>(*agnKernels[device]);
+    auto assignOp = KernelFunctor< Buffer, KParam, Buffer, KParam, AssignKernelParam_t,
+                                   Buffer, Buffer, Buffer, Buffer, int, int>(*entry.ker);
 
     assignOp(EnqueueArgs(getQueue(), global, local),
-            *out.data, out.info, *in.data, in.info, p,
-            *bPtr[0], *bPtr[1], *bPtr[2], *bPtr[3], blk_x, blk_y);
+             *out.data, out.info, *in.data, in.info, p,
+             *bPtr[0], *bPtr[1], *bPtr[2], *bPtr[3], blk_x, blk_y);
 
     CL_DEBUG_FINISH(getQueue());
 }
-
 }
-
 }
-
diff --git a/src/backend/opencl/kernel/bilateral.hpp b/src/backend/opencl/kernel/bilateral.hpp
index 46eabab6c6..1d93de83ba 100644
--- a/src/backend/opencl/kernel/bilateral.hpp
+++ b/src/backend/opencl/kernel/bilateral.hpp
@@ -12,9 +12,8 @@
 #include <program.hpp>
 #include <traits.hpp>
 #include <string>
-#include <mutex>
-#include <map>
 #include <algorithm>
+#include <cache.hpp>
 #include <dispatch.hpp>
 #include <Param.hpp>
 #include <debug_opencl.hpp>
@@ -31,56 +30,53 @@ using std::string;
 
 namespace opencl
 {
-
 namespace kernel
 {
-
 static const int THREADS_X = 16;
 static const int THREADS_Y = 16;
 
 template<typename inType, typename outType, bool isColor>
 void bilateral(Param out, const Param in, float s_sigma, float c_sigma)
 {
-    static std::once_flag compileFlags[DeviceManager::MAX_DEVICES];
-    static std::map<int, Program*>  bilProgs;
-    static std::map<int, Kernel*> bilKernels;
+    std::string refName = std::string("bilateral_") +
+        std::string(dtype_traits<inType>::getName()) +
+        std::string(dtype_traits<outType>::getName()) +
+        std::to_string(isColor);
 
     int device = getActiveDeviceId();
-
-    std::call_once( compileFlags[device], [device] () {
-            bool use_native_exp = (getActivePlatform() != AFCL_PLATFORM_POCL
-                                && getActivePlatform() != AFCL_PLATFORM_APPLE);
-            std::ostringstream options;
-            options << " -D inType=" << dtype_traits<inType>::getName()
-                    << " -D outType=" << dtype_traits<outType>::getName();
-            if (std::is_same<inType, double>::value ||
+    kc_entry_t entry = kernelCache(device, refName);
+
+    if (entry.prog==0 && entry.ker==0) {
+        bool use_native_exp = (getActivePlatform() != AFCL_PLATFORM_POCL
+                && getActivePlatform() != AFCL_PLATFORM_APPLE);
+        std::ostringstream options;
+        options << " -D inType=" << dtype_traits<inType>::getName()
+            << " -D outType=" << dtype_traits<outType>::getName();
+        if (std::is_same<inType, double>::value ||
                 std::is_same<inType, cdouble>::value) {
-                    options << " -D USE_DOUBLE";
-            }
-            options << " -D USE_NATIVE_EXP=" << (int)use_native_exp;
-
-            Program prog;
-            buildProgram(prog, bilateral_cl, bilateral_cl_len, options.str());
-            bilProgs[device] = new Program(prog);
-
-                bilKernels[device] = new Kernel(*bilProgs[device], "bilateral");
-        });
+            options << " -D USE_DOUBLE";
+        }
+        options << " -D USE_NATIVE_EXP=" << (int)use_native_exp;
+
+        const char* ker_strs[] = {bilateral_cl};
+        const int   ker_lens[] = {bilateral_cl_len};
+        Program prog;
+        buildProgram(prog, 1, ker_strs, ker_lens, options.str());
+        entry.prog = new Program(prog);
+        entry.ker  = new Kernel(*entry.prog, "bilateral");
+
+        addKernelToCache(device, refName, entry);
+    }
 
-    auto bilateralOp = KernelFunctor<Buffer, KParam,
-                                    Buffer, KParam,
-                                    LocalSpaceArg,
-                                    LocalSpaceArg,
-                                    float, float,
-                                    int, int, int
-                                  >(*bilKernels[device]);
+    auto bilateralOp = KernelFunctor< Buffer, KParam, Buffer, KParam, LocalSpaceArg, LocalSpaceArg,
+                                      float, float, int, int, int >(*entry.ker);
 
     NDRange local(THREADS_X, THREADS_Y);
 
     int blk_x = divup(in.info.dims[0], THREADS_X);
     int blk_y = divup(in.info.dims[1], THREADS_Y);
 
-    NDRange global(blk_x*in.info.dims[2]*THREADS_X,
-                    blk_y*in.info.dims[3]*THREADS_Y);
+    NDRange global(blk_x*in.info.dims[2]*THREADS_X, blk_y*in.info.dims[3]*THREADS_Y);
 
     // calculate local memory size
     int radius = (int)std::max(s_sigma * 1.5f, 1.f);
@@ -100,7 +96,5 @@ void bilateral(Param out, const Param in, float s_sigma, float c_sigma)
 
     CL_DEBUG_FINISH(getQueue());
 }
-
 }
-
 }
diff --git a/src/backend/opencl/kernel/convolve/conv_common.hpp b/src/backend/opencl/kernel/convolve/conv_common.hpp
index 9ea8e23e70..f74a22e50c 100644
--- a/src/backend/opencl/kernel/convolve/conv_common.hpp
+++ b/src/backend/opencl/kernel/convolve/conv_common.hpp
@@ -12,8 +12,6 @@
 
 #include <kernel_headers/convolve.hpp>
 
-#include <map>
-#include <mutex>
 #include <string>
 #include <Param.hpp>
 #include <types.hpp>
@@ -23,6 +21,7 @@
 #include <dispatch.hpp>
 #include <platform.hpp>
 #include <debug_opencl.hpp>
+#include <cache.hpp>
 
 using cl::Buffer;
 using cl::Program;
@@ -33,10 +32,8 @@ using std::string;
 
 namespace opencl
 {
-
 namespace kernel
 {
-
 static const int THREADS   = 256;
 
 static const int THREADS_X = 16;
@@ -97,39 +94,37 @@ void prepareKernelArgs(conv_kparam_t& param, dim_t *oDims,
 template<typename T, typename aT, int bDim, bool expand>
 void convNHelper(const conv_kparam_t& param, Param& out, const Param& signal, const Param& filter)
 {
-    static std::once_flag  compileFlags[DeviceManager::MAX_DEVICES];
-    static std::map<int, Program*> convProgs;
-    static std::map<int, Kernel*>  convKernels;
+    std::string ref_name = std::string("convolveND_") +
+        std::string(dtype_traits<T>::getName()) + std::string(dtype_traits<aT>::getName()) +
+        std::to_string(bDim) + std::to_string(expand);
 
     int device = getActiveDeviceId();
 
-    std::call_once( compileFlags[device], [device] () {
-                std::ostringstream options;
-                options << " -D T=" << dtype_traits<T>::getName()
-                        << " -D accType="<< dtype_traits<aT>::getName()
-                        << " -D BASE_DIM="<< bDim
-                        << " -D EXPAND=" << expand;
-                if (std::is_same<T, double>::value ||
-                    std::is_same<T, cdouble>::value) {
-                    options << " -D USE_DOUBLE";
-                }
-                Program prog;
-                buildProgram(prog, convolve_cl, convolve_cl_len, options.str());
-                convProgs[device]   = new Program(prog);
-                convKernels[device] = new Kernel(*convProgs[device], "convolve");
-            });
-
-    auto convOp = cl::KernelFunctor<Buffer, KParam, Buffer, KParam,
-                                    cl::LocalSpaceArg, Buffer, KParam,
-                                    int, int,
-                                    int, int, int,
-                                    int, int, int
-                                    >(*convKernels[device]);
+    kc_entry_t entry = kernelCache(device, ref_name);
+
+    if (entry.prog==0 && entry.ker==0) {
+        std::ostringstream options;
+        options << " -D T="         << dtype_traits<T>::getName()
+                << " -D accType="   << dtype_traits<aT>::getName()
+                << " -D BASE_DIM="  << bDim
+                << " -D EXPAND="    << expand;
+        if (std::is_same<T, double>::value || std::is_same<T, cdouble>::value)
+            options << " -D USE_DOUBLE";
+        Program prog;
+        buildProgram(prog, convolve_cl, convolve_cl_len, options.str());
+        entry.prog   = new Program(prog);
+        entry.ker = new Kernel(*entry.prog, "convolve");
+
+        addKernelToCache(device, ref_name, entry);
+    }
+
+    auto convOp = cl::KernelFunctor<Buffer, KParam, Buffer, KParam, cl::LocalSpaceArg, Buffer, KParam,
+                                    int, int, int, int, int, int, int, int >(*entry.ker);
 
     convOp(EnqueueArgs(getQueue(), param.global, param.local),
-            *out.data, out.info, *signal.data, signal.info, cl::Local(param.loc_size),
-            *param.impulse, filter.info, param.nBBS0, param.nBBS1,
-            param.o[0], param.o[1], param.o[2], param.s[0], param.s[1], param.s[2]);
+           *out.data, out.info, *signal.data, signal.info, cl::Local(param.loc_size),
+           *param.impulse, filter.info, param.nBBS0, param.nBBS1,
+           param.o[0], param.o[1], param.o[2], param.s[0], param.s[1], param.s[2]);
 }
 
 template<typename T, typename aT, bool expand>
@@ -140,7 +135,5 @@ void conv2(conv_kparam_t& p, Param& out, const Param& sig, const Param& filt);
 
 template<typename T, typename aT, bool expand>
 void conv3(conv_kparam_t& p, Param& out, const Param& sig, const Param& filt);
-
 }
-
 }
diff --git a/src/backend/opencl/kernel/diagonal.hpp b/src/backend/opencl/kernel/diagonal.hpp
index bf6acce9af..70659e580a 100644
--- a/src/backend/opencl/kernel/diagonal.hpp
+++ b/src/backend/opencl/kernel/diagonal.hpp
@@ -14,8 +14,7 @@
 #include <dispatch.hpp>
 #include <Param.hpp>
 #include <debug_opencl.hpp>
-#include <map>
-#include <mutex>
+#include <cache.hpp>
 #include <math.hpp>
 #include "config.hpp"
 
@@ -30,89 +29,87 @@ using af::scalar_to_option;
 
 namespace opencl
 {
-
 namespace kernel
 {
-
-    template<typename T>
-    static void diagCreate(Param out, Param in, int num)
-    {
-        static std::once_flag compileFlags[DeviceManager::MAX_DEVICES];
-        static std::map<int, Program*>   diagCreateProgs;
-        static std::map<int, Kernel*>  diagCreateKernels;
-
-        int device = getActiveDeviceId();
-
-        std::call_once( compileFlags[device], [device] () {
-                std::ostringstream options;
-                options << " -D T="    << dtype_traits<T>::getName()
-                        << " -D ZERO=(T)(" << scalar_to_option(scalar<T>(0)) << ")";
-                if (std::is_same<T, double>::value ||
-                    std::is_same<T, cdouble>::value) {
-                    options << " -D USE_DOUBLE";
-                }
-                Program prog;
-                buildProgram(prog, diag_create_cl, diag_create_cl_len, options.str());
-                diagCreateProgs[device]   = new Program(prog);
-                diagCreateKernels[device] = new Kernel(*diagCreateProgs[device],
-                                                        "diagCreateKernel");
-            });
-
-        NDRange local(32, 8);
-        int groups_x = divup(out.info.dims[0], local[0]);
-        int groups_y = divup(out.info.dims[1], local[1]);
-        NDRange global(groups_x * local[0] * out.info.dims[2],
-                        groups_y * local[1]);
-
-        auto diagCreateOp = KernelFunctor<Buffer, const KParam,
-                                        Buffer, const KParam,
-                                        int, int> (*diagCreateKernels[device]);
-
-        diagCreateOp(EnqueueArgs(getQueue(), global, local),
-                      *(out.data), out.info, *(in.data), in.info, num, groups_x);
-        CL_DEBUG_FINISH(getQueue());
+template<typename T>
+std::string generateOptionsString()
+{
+    std::ostringstream options;
+    options << " -D T="    << dtype_traits<T>::getName()
+            << " -D ZERO=(T)(" << scalar_to_option(scalar<T>(0)) << ")";
+    if (std::is_same<T, double>::value || std::is_same<T, cdouble>::value) {
+        options << " -D USE_DOUBLE";
     }
+    return options.str();
+}
+
+template<typename T>
+static void diagCreate(Param out, Param in, int num)
+{
+    std::string refName = std::string("diagCreateKernel_") + std::string(dtype_traits<T>::getName());
 
-    template<typename T>
-    static void diagExtract(Param out, Param in, int num)
-    {
-        static std::once_flag compileFlags[DeviceManager::MAX_DEVICES];
-        static std::map<int, Program*>   diagExtractProgs;
-        static std::map<int, Kernel*>  diagExtractKernels;
-
-        int device = getActiveDeviceId();
-
-        std::call_once( compileFlags[device], [device] () {
-                std::ostringstream options;
-                options << " -D T="    << dtype_traits<T>::getName()
-                        << " -D ZERO=(T)(" << scalar_to_option(scalar<T>(0)) << ")";
-                if (std::is_same<T, double>::value ||
-                    std::is_same<T, cdouble>::value) {
-                    options << " -D USE_DOUBLE";
-                }
-                Program prog;
-                buildProgram(prog, diag_extract_cl, diag_extract_cl_len, options.str());
-                diagExtractProgs[device]   = new Program(prog);
-                diagExtractKernels[device] = new Kernel(*diagExtractProgs[device],
-                                                        "diagExtractKernel");
-            });
-
-        NDRange local(256, 1);
-        int groups_x = divup(out.info.dims[0], local[0]);
-        int groups_z = out.info.dims[2];
-        NDRange global(groups_x * local[0],
-                        groups_z * local[1] * out.info.dims[3]);
-
-        auto diagExtractOp = KernelFunctor<Buffer, const KParam,
-                                          Buffer, const KParam,
-                                          int, int> (*diagExtractKernels[device]);
-
-        diagExtractOp(EnqueueArgs(getQueue(), global, local),
-                      *(out.data), out.info, *(in.data), in.info, num, groups_z);
-        CL_DEBUG_FINISH(getQueue());
+    int device = getActiveDeviceId();
+    kc_entry_t entry = kernelCache(device, refName);
 
+    if (entry.prog==0 && entry.ker==0) {
+        std::string options = generateOptionsString<T>();
+        const char* ker_strs[] = {diag_create_cl};
+        const int   ker_lens[] = {diag_create_cl_len};
+        Program prog;
+        buildProgram(prog, 1, ker_strs, ker_lens, options);
+        entry.prog = new Program(prog);
+        entry.ker  = new Kernel(*entry.prog, "diagCreateKernel");
+
+        addKernelToCache(device, refName, entry);
     }
 
+    NDRange local(32, 8);
+    int groups_x = divup(out.info.dims[0], local[0]);
+    int groups_y = divup(out.info.dims[1], local[1]);
+    NDRange global(groups_x * local[0] * out.info.dims[2], groups_y * local[1]);
+
+    auto diagCreateOp = KernelFunctor< Buffer, const KParam, Buffer, const KParam,
+                                       int, int > (*entry.ker);
+
+    diagCreateOp(EnqueueArgs(getQueue(), global, local),
+                 *(out.data), out.info, *(in.data), in.info, num, groups_x);
+
+    CL_DEBUG_FINISH(getQueue());
 }
 
+template<typename T>
+static void diagExtract(Param out, Param in, int num)
+{
+    std::string refName = std::string("diagExtractKernel_") + std::string(dtype_traits<T>::getName());
+
+    int device = getActiveDeviceId();
+    kc_entry_t entry = kernelCache(device, refName);
+
+    if (entry.prog==0 && entry.ker==0) {
+        std::string options = generateOptionsString<T>();
+        const char* ker_strs[] = {diag_extract_cl};
+        const int   ker_lens[] = {diag_extract_cl_len};
+        Program prog;
+        buildProgram(prog, 1, ker_strs, ker_lens, options);
+        entry.prog = new Program(prog);
+        entry.ker  = new Kernel(*entry.prog, "diagExtractKernel");
+
+        addKernelToCache(device, refName, entry);
+    }
+
+    NDRange local(256, 1);
+    int groups_x = divup(out.info.dims[0], local[0]);
+    int groups_z = out.info.dims[2];
+    NDRange global(groups_x * local[0], groups_z * local[1] * out.info.dims[3]);
+
+    auto diagExtractOp = KernelFunctor< Buffer, const KParam, Buffer, const KParam,
+                                        int, int > (*entry.ker);
+
+    diagExtractOp(EnqueueArgs(getQueue(), global, local),
+                  *(out.data), out.info, *(in.data), in.info, num, groups_z);
+
+    CL_DEBUG_FINISH(getQueue());
+
+}
+}
 }
diff --git a/src/backend/opencl/kernel/diff.hpp b/src/backend/opencl/kernel/diff.hpp
index 1445829c19..438ad13354 100644
--- a/src/backend/opencl/kernel/diff.hpp
+++ b/src/backend/opencl/kernel/diff.hpp
@@ -12,8 +12,7 @@
 #include <program.hpp>
 #include <traits.hpp>
 #include <string>
-#include <mutex>
-#include <map>
+#include <cache.hpp>
 #include <dispatch.hpp>
 #include <Param.hpp>
 #include <debug_opencl.hpp>
@@ -28,58 +27,61 @@ using std::string;
 
 namespace opencl
 {
-    namespace kernel
-    {
-        static const int TX = 16;
-        static const int TY = 16;
+namespace kernel
+{
+static const int TX = 16;
+static const int TY = 16;
 
-        template<typename T, unsigned dim, bool isDiff2>
-        void diff(Param out, const Param in, const unsigned indims)
-        {
-            static std::once_flag compileFlags[DeviceManager::MAX_DEVICES];
-            static std::map<int, Program*>   diffProgs;
-            static std::map<int, Kernel*>  diffKernels;
+template<typename T, unsigned dim, bool isDiff2>
+void diff(Param out, const Param in, const unsigned indims)
+{
+    std::string refName = std::string("diff_kernel_") +
+        std::string(dtype_traits<T>::getName()) +
+        std::to_string(dim) +
+        std::to_string(isDiff2);
 
-            int device = getActiveDeviceId();
+    int device = getActiveDeviceId();
+    kc_entry_t entry = kernelCache(device, refName);
 
-            std::call_once( compileFlags[device], [device] () {
-                std::ostringstream options;
-                options << " -D T="        << dtype_traits<T>::getName()
-                        << " -D DIM="      << dim
-                        << " -D isDiff2=" << isDiff2;
-                if (std::is_same<T, double>::value ||
-                    std::is_same<T, cdouble>::value) {
-                    options << " -D USE_DOUBLE";
-                }
-                Program prog;
-                buildProgram(prog, diff_cl, diff_cl_len, options.str());
-                diffProgs[device]   = new Program(prog);
-                diffKernels[device] = new Kernel(*diffProgs[device], "diff_kernel");
-            });
+    if (entry.prog==0 && entry.ker==0) {
+        std::ostringstream options;
+        options << " -D T="        << dtype_traits<T>::getName()
+                << " -D DIM="      << dim
+                << " -D isDiff2=" << isDiff2;
+        if (std::is_same<T, double>::value ||
+            std::is_same<T, cdouble>::value) {
+            options << " -D USE_DOUBLE";
+        }
 
-            auto diffOp = KernelFunctor<Buffer, const Buffer, const KParam, const KParam,
-                                      const int, const int, const int>
-                                      (*diffKernels[device]);
+        const char* ker_strs[] = {diff_cl};
+        const int   ker_lens[] = {diff_cl_len};
+        Program prog;
+        buildProgram(prog, 1, ker_strs, ker_lens, options.str());
+        entry.prog = new Program(prog);
+        entry.ker  = new Kernel(*entry.prog, "diff_kernel");
 
-            NDRange local(TX, TY, 1);
-            if(dim == 0 && indims == 1) {
-                local = NDRange(TX * TY, 1, 1);
-            }
+        addKernelToCache(device, refName, entry);
+    }
 
-            int blocksPerMatX = divup(out.info.dims[0], local[0]);
-            int blocksPerMatY = divup(out.info.dims[1], local[1]);
-            NDRange global(local[0] * blocksPerMatX * out.info.dims[2],
-                            local[1] * blocksPerMatY * out.info.dims[3],
-                            1);
+    auto diffOp = KernelFunctor< Buffer, const Buffer, const KParam, const KParam,
+                                 const int, const int, const int> (*entry.ker);
 
-            const int oElem = out.info.dims[0] * out.info.dims[1]
-                                  * out.info.dims[2] * out.info.dims[3];
+    NDRange local(TX, TY, 1);
+    if(dim == 0 && indims == 1) {
+        local = NDRange(TX * TY, 1, 1);
+    }
 
-            diffOp(EnqueueArgs(getQueue(), global, local),
-                    *out.data, *in.data, out.info, in.info,
-                    oElem, blocksPerMatX, blocksPerMatY);
+    int blocksPerMatX = divup(out.info.dims[0], local[0]);
+    int blocksPerMatY = divup(out.info.dims[1], local[1]);
+    NDRange global(local[0] * blocksPerMatX * out.info.dims[2],
+                   local[1] * blocksPerMatY * out.info.dims[3], 1);
 
-            CL_DEBUG_FINISH(getQueue());
-        }
-    }
+    const int oElem = out.info.dims[0] * out.info.dims[1] * out.info.dims[2] * out.info.dims[3];
+
+    diffOp(EnqueueArgs(getQueue(), global, local),
+           *out.data, *in.data, out.info, in.info, oElem, blocksPerMatX, blocksPerMatY);
+
+    CL_DEBUG_FINISH(getQueue());
+}
+}
 }
diff --git a/src/backend/opencl/kernel/exampleFunction.hpp b/src/backend/opencl/kernel/exampleFunction.hpp
index 2af050e33b..acad35b831 100644
--- a/src/backend/opencl/kernel/exampleFunction.hpp
+++ b/src/backend/opencl/kernel/exampleFunction.hpp
@@ -18,8 +18,12 @@
 // Following c++ standard library headers are needed to maintain
 // OpenCL cl::Kernel & cl::Program objects
 #include <string>
-#include <mutex>
-#include <map>
+
+#include <cache.hpp>                    // Has the definitions of functions such as the following
+                                        // used in caching and fetching kernels.
+                                        // * kernelCache - used to fetch existing kernel from cache
+                                        // if any
+                                        // * addKernelToCache - push new kernels into cache
 
 #include <dispatch.hpp>                 // common utility header for CUDA & OpenCL backends
                                         // has the divup macro
@@ -43,54 +47,51 @@ using std::string;
 
 namespace opencl
 {
-
 namespace kernel
 {
-
 static const int THREADS_X = 16;
 static const int THREADS_Y = 16;
 
 template<typename T>
 void exampleFunc(Param c, const Param a, const Param b, const af_someenum_t p)
 {
-    static std::once_flag compileFlags[DeviceManager::MAX_DEVICES];
-    static std::map<int, Program*>  egProgs;
-    static std::map<int, Kernel*> egKernels;
+    std::string refName =
+        std::string("example_") + //<kernel_function_name>_
+        std::string(dtype_traits<T>::getName());
+        // std::string("encode template parameters one after one");
+        // If you have numericals, you can use std::to_string to convert
+        // them into std::strings
 
     int device = getActiveDeviceId();
-
-    // std::call_once is used to ensure OpenCL kernels
-    // are compiled only once for any given device and combination
-    // of template parameters to this kernel wrapper function 'exampleFunc<T>'
-    std::call_once( compileFlags[device], [device] () {
-
-            std::ostringstream options;
-            options << " -D T=" << dtype_traits<T>::getName();
-            // You can pass any template parameters as compile options
-            // to kernel the compilation step. This is equivalent of
-            // having templated kernels in CUDA
-
-            // The following option is passed to kernel compilation
-            // if template parameter T is double or complex double
-            // to enable FP64 extension
-            if (std::is_same<T, double>::value ||
+    kc_entry_t entry = kernelCache(device, refName);
+
+    // Make sure OpenCL kernel isn't already available before
+    // compiling for given device and combination of template
+    // parameters to this kernel wrapper function 'exampleFunc<T>'
+    if (entry.prog==0 && entry.ker==0) {
+        std::ostringstream options;
+        options << " -D T=" << dtype_traits<T>::getName();
+        // You can pass any template parameters as compile options
+        // to kernel the compilation step. This is equivalent of
+        // having templated kernels in CUDA
+
+        // The following option is passed to kernel compilation
+        // if template parameter T is double or complex double
+        // to enable FP64 extension
+        if (std::is_same<T, double>::value ||
                 std::is_same<T, cdouble>::value) {
-                options << " -D USE_DOUBLE";
-            }
+            options << " -D USE_DOUBLE";
+        }
 
-            Program prog;
-            // below helper function 'buildProgram' uses the option string
-            // we just created and compiles the kernel string
-            // 'example_cl' which was created by our opencl kernel code obfuscation
-            // stage
-            buildProgram(prog, example_cl, example_cl_len, options.str());
+        const char *ker_strs[] = {example_cl};
+        const int   ker_lens[] = {example_cl_len};
+        Program prog;
+        buildProgram(prog, 1, ker_strs, ker_lens, options.str());
+        entry.prog = new Program(prog);
+        entry.ker = new Kernel(*entry.prog, "example");
 
-            // create a cl::Program object on heap
-            egProgs[device]   = new Program(prog);
-
-            // create a cl::Kernel object on heap
-            egKernels[device] = new Kernel(*egProgs[device], "example");
-        });
+        addKernelToCache(device, refName, entry);
+    }
 
     // configure work group parameters
     NDRange local(THREADS_X, THREADS_Y);
@@ -104,18 +105,16 @@ void exampleFunc(Param c, const Param a, const Param b, const af_someenum_t p)
     // create a kernel functor from the cl::Kernel object
     // corresponding to the device on which current execution
     // is happending.
-    auto exampleFuncOp = KernelFunctor<Buffer, KParam, Buffer, KParam,
-                                        Buffer, KParam, int>(*egKernels[device]);
+    auto exampleFuncOp = KernelFunctor< Buffer, KParam, Buffer, KParam,
+                                        Buffer, KParam, int>(*entry.ker);
 
     // launch the kernel
     exampleFuncOp(EnqueueArgs(getQueue(), global, local),
-                *c.data, c.info, *a.data, a.info, *b.data, b.info, (int)p);
+                  *c.data, c.info, *a.data, a.info, *b.data, b.info, (int)p);
 
     // Below Macro activates validations ONLY in DEBUG
     // mode as its name indicates
     CL_DEBUG_FINISH(getQueue());
 }
-
 }
-
 }
diff --git a/src/backend/opencl/kernel/fftconvolve.hpp b/src/backend/opencl/kernel/fftconvolve.hpp
index aa1449d1fc..1204047fe5 100644
--- a/src/backend/opencl/kernel/fftconvolve.hpp
+++ b/src/backend/opencl/kernel/fftconvolve.hpp
@@ -11,13 +11,13 @@
 #include <backend.hpp>
 #include <program.hpp>
 #include <dispatch.hpp>
+#include <cache.hpp>
 #include <err_opencl.hpp>
 #include <debug_opencl.hpp>
 #include <kernel_headers/fftconvolve_pack.hpp>
 #include <kernel_headers/fftconvolve_multiply.hpp>
 #include <kernel_headers/fftconvolve_reorder.hpp>
 #include <memory.hpp>
-#include <map>
 
 using cl::Buffer;
 using cl::Program;
@@ -29,10 +29,8 @@ using cl::NDRange;
 
 namespace opencl
 {
-
 namespace kernel
 {
-
 static const int THREADS = 256;
 
 void calcParamSizes(Param& sig_tmp,
@@ -81,33 +79,37 @@ void packDataHelper(Param packed,
                     const int baseDim,
                     AF_BATCH_KIND kind)
 {
-    static std::once_flag     compileFlags[DeviceManager::MAX_DEVICES];
-    static std::map<int, Program*>  fftconvolveProgs;
-    static std::map<int, Kernel*>   pdKernel;
-    static std::map<int, Kernel*>   paKernel;
+    std::string refName =
+        std::string("pack_data_") +
+        std::string(dtype_traits<convT>::getName()) +
+        std::string(dtype_traits<T>::getName()) +
+        std::to_string(isDouble);
 
     int device = getActiveDeviceId();
+    kc_entry_t pdkEntry = kernelCache(device, refName);
 
-    std::call_once( compileFlags[device], [device] () {
+    if (pdkEntry.prog==0 && pdkEntry.ker==0) {
+        std::ostringstream options;
 
-            std::ostringstream options;
-            options << " -D T=" << dtype_traits<T>::getName();
+        options << " -D T=" << dtype_traits<T>::getName();
 
-            if ((af_dtype) dtype_traits<convT>::af_type == c32) {
-                options << " -D CONVT=float";
-            }
-            else if ((af_dtype) dtype_traits<convT>::af_type == c64 && isDouble) {
-                options << " -D CONVT=double"
-                        << " -D USE_DOUBLE";
-            }
+        if ((af_dtype) dtype_traits<convT>::af_type == c32) {
+            options << " -D CONVT=float";
+        }
+        else if ((af_dtype) dtype_traits<convT>::af_type == c64 && isDouble) {
+            options << " -D CONVT=double"
+                << " -D USE_DOUBLE";
+        }
 
-            cl::Program prog;
-            buildProgram(prog, fftconvolve_pack_cl, fftconvolve_pack_cl_len, options.str());
-            fftconvolveProgs[device] = new Program(prog);
+        const char* ker_strs[] = {fftconvolve_pack_cl};
+        const int   ker_lens[] = {fftconvolve_pack_cl_len};
+        Program prog;
+        buildProgram(prog, 1, ker_strs, ker_lens, options.str());
+        pdkEntry.prog = new Program(prog);
+        pdkEntry.ker  = new Kernel(*pdkEntry.prog, "pack_data");
 
-            pdKernel[device] = new Kernel(*fftconvolveProgs[device], "pack_data");
-            paKernel[device] = new Kernel(*fftconvolveProgs[device], "pad_array");
-        });
+        addKernelToCache(device, refName, pdkEntry);
+    }
 
     Param sig_tmp, filter_tmp;
     calcParamSizes(sig_tmp, filter_tmp, packed, sig, filter, baseDim, kind);
@@ -127,25 +129,53 @@ void packDataHelper(Param packed,
 
     // Pack signal in a complex matrix where first dimension is half the input
     // (allows faster FFT computation) and pad array to a power of 2 with 0s
-    auto pdOp = KernelFunctor<Buffer, KParam,
-                            Buffer, KParam,
-                            const int, const int> (*pdKernel[device]);
+    auto pdOp = KernelFunctor< Buffer, KParam, Buffer, KParam, const int, const int > (*pdkEntry.ker);
 
     pdOp(EnqueueArgs(getQueue(), global, local),
-          *sig_tmp.data, sig_tmp.info, *sig.data, sig.info,
-          sig_half_d0, sig_half_d0_odd);
+         *sig_tmp.data, sig_tmp.info, *sig.data, sig.info, sig_half_d0, sig_half_d0_odd);
+
     CL_DEBUG_FINISH(getQueue());
 
+    refName =
+        std::string("pack_array_") +
+        std::string(dtype_traits<convT>::getName()) +
+        std::string(dtype_traits<T>::getName()) +
+        std::to_string(isDouble);
+
+    kc_entry_t pakEntry = kernelCache(device, refName);
+
+    if (pakEntry.prog==0 && pakEntry.ker==0) {
+        std::ostringstream options;
+
+        options << " -D T=" << dtype_traits<T>::getName();
+
+        if ((af_dtype) dtype_traits<convT>::af_type == c32) {
+            options << " -D CONVT=float";
+        }
+        else if ((af_dtype) dtype_traits<convT>::af_type == c64 && isDouble) {
+            options << " -D CONVT=double"
+                << " -D USE_DOUBLE";
+        }
+
+        const char* ker_strs[] = {fftconvolve_pack_cl};
+        const int   ker_lens[] = {fftconvolve_pack_cl_len};
+        Program prog;
+        buildProgram(prog, 1, ker_strs, ker_lens, options.str());
+        pakEntry.prog = new Program(prog);
+        pakEntry.ker  = new Kernel(*pakEntry.prog, "pad_array");
+
+        addKernelToCache(device, refName, pakEntry);
+    }
+
     blocks = divup(filter_packed_elem, THREADS);
     global = NDRange(blocks * THREADS);
 
     // Pad filter array with 0s
-    auto paOp = KernelFunctor<Buffer, KParam,
-                            Buffer, KParam> (*paKernel[device]);
+    auto paOp = KernelFunctor< Buffer, KParam, Buffer, KParam > (*pakEntry.ker);
 
     paOp(EnqueueArgs(getQueue(), global, local),
-          *filter_tmp.data, filter_tmp.info,
-          *filter.data, filter.info);
+         *filter_tmp.data, filter_tmp.info, *filter.data, filter.info);
+
     CL_DEBUG_FINISH(getQueue());
 }
 
@@ -156,38 +186,40 @@ void complexMultiplyHelper(Param packed,
                            const int baseDim,
                            AF_BATCH_KIND kind)
 {
-    static std::once_flag     compileFlags[DeviceManager::MAX_DEVICES];
-    static std::map<int, Program*> fftconvolveProgs;
-    static std::map<int, Kernel*>  cmKernel;
+    std::string refName =
+        std::string("complex_multiply_") +
+        std::string(dtype_traits<convT>::getName()) +
+        std::string(dtype_traits<T>::getName()) +
+        std::to_string(isDouble);
 
     int device = getActiveDeviceId();
+    kc_entry_t entry = kernelCache(device, refName);
+
+    if (entry.prog==0 && entry.ker==0) {
+        std::ostringstream options;
+
+        options << " -D T=" << dtype_traits<T>::getName()
+            << " -D AF_BATCH_NONE=" << (int)AF_BATCH_NONE
+            << " -D AF_BATCH_LHS="  << (int)AF_BATCH_LHS
+            << " -D AF_BATCH_RHS="  << (int)AF_BATCH_RHS
+            << " -D AF_BATCH_SAME=" << (int)AF_BATCH_SAME;
+
+        if ((af_dtype) dtype_traits<convT>::af_type == c32) {
+            options << " -D CONVT=float";
+        } else if ((af_dtype) dtype_traits<convT>::af_type == c64 && isDouble) {
+            options << " -D CONVT=double"
+                    << " -D USE_DOUBLE";
+        }
+
+        const char* ker_strs[] = {fftconvolve_multiply_cl};
+        const int   ker_lens[] = {fftconvolve_multiply_cl_len};
+        Program prog;
+        buildProgram(prog, 1, ker_strs, ker_lens, options.str());
+        entry.prog = new Program(prog);
+        entry.ker  = new Kernel(*entry.prog, "complex_multiply");
 
-    std::call_once( compileFlags[device], [device] () {
-
-            std::ostringstream options;
-            options << " -D T=" << dtype_traits<T>::getName()
-                    << " -D AF_BATCH_NONE=" << (int)AF_BATCH_NONE
-                    << " -D AF_BATCH_LHS="  << (int)AF_BATCH_LHS
-                    << " -D AF_BATCH_RHS="  << (int)AF_BATCH_RHS
-                    << " -D AF_BATCH_SAME=" << (int)AF_BATCH_SAME;
-
-            if ((af_dtype) dtype_traits<convT>::af_type == c32) {
-                options << " -D CONVT=float";
-            }
-            else if ((af_dtype) dtype_traits<convT>::af_type == c64 && isDouble) {
-                options << " -D CONVT=double"
-                        << " -D USE_DOUBLE";
-            }
-
-            cl::Program prog;
-            buildProgram(prog,
-                          fftconvolve_multiply_cl,
-                          fftconvolve_multiply_cl_len,
-                          options.str());
-            fftconvolveProgs[device] = new Program(prog);
-
-            cmKernel[device] = new Kernel(*fftconvolveProgs[device], "complex_multiply");
-        });
+        addKernelToCache(device, refName, entry);
+    }
 
     Param sig_tmp, filter_tmp;
     calcParamSizes(sig_tmp, filter_tmp, packed, sig, filter, baseDim, kind);
@@ -203,16 +235,13 @@ void complexMultiplyHelper(Param packed,
     NDRange global(blocks * THREADS);
 
     // Multiply filter and signal FFT arrays
-    auto cmOp = KernelFunctor<Buffer, KParam,
-                            Buffer, KParam,
-                            Buffer, KParam,
-                            const int, const int> (*cmKernel[device]);
+    auto cmOp = KernelFunctor< Buffer, KParam, Buffer, KParam, Buffer, KParam,
+                               const int, const int > (*entry.ker);
 
     cmOp(EnqueueArgs(getQueue(), global, local),
-          *packed.data, packed.info,
-          *sig_tmp.data, sig_tmp.info,
-          *filter_tmp.data, filter_tmp.info,
-          mul_elem, (int)kind);
+         *packed.data, packed.info, *sig_tmp.data, sig_tmp.info,
+         *filter_tmp.data, filter_tmp.info, mul_elem, (int)kind);
+
     CL_DEBUG_FINISH(getQueue());
 }
 
@@ -224,42 +253,46 @@ void reorderOutputHelper(Param out,
                          const int baseDim,
                          AF_BATCH_KIND kind)
 {
-    static std::once_flag     compileFlags[DeviceManager::MAX_DEVICES];
-    static std::map<int, Program*> fftconvolveProgs;
-    static std::map<int, Kernel*>  roKernel;
+    std::string refName =
+        std::string("reorder_output_") +
+        std::string(dtype_traits<T>::getName()) +
+        std::string(dtype_traits<convT>::getName()) +
+        std::to_string(isDouble) +
+        std::to_string(roundOut) +
+        std::to_string(expand);
 
-    int fftScale = 1;
+    int device = getActiveDeviceId();
+    kc_entry_t entry = kernelCache(device, refName);
 
-    // Calculate the scale by which to divide clFFT results
-    for (int k = 0; k < baseDim; k++)
-        fftScale *= packed.info.dims[k];
+    if (entry.prog==0 && entry.ker==0) {
+        std::ostringstream options;
 
-    int device = getActiveDeviceId();
+        options << " -D T=" << dtype_traits<T>::getName()
+            << " -D ROUND_OUT=" << (int)roundOut
+            << " -D EXPAND=" << (int)expand;
 
-    std::call_once( compileFlags[device], [device] () {
+        if ((af_dtype) dtype_traits<convT>::af_type == c32) {
+            options << " -D CONVT=float";
+        } else if ((af_dtype) dtype_traits<convT>::af_type == c64 && isDouble) {
+            options << " -D CONVT=double"
+                << " -D USE_DOUBLE";
+        }
 
-            std::ostringstream options;
-            options << " -D T=" << dtype_traits<T>::getName()
-                    << " -D ROUND_OUT=" << (int)roundOut
-                    << " -D EXPAND=" << (int)expand;
+        const char* ker_strs[] = {fftconvolve_reorder_cl};
+        const int   ker_lens[] = {fftconvolve_reorder_cl_len};
+        Program prog;
+        buildProgram(prog, 1, ker_strs, ker_lens, options.str());
+        entry.prog = new Program(prog);
+        entry.ker  = new Kernel(*entry.prog, "reorder_output");
 
-            if ((af_dtype) dtype_traits<convT>::af_type == c32) {
-                options << " -D CONVT=float";
-            }
-            else if ((af_dtype) dtype_traits<convT>::af_type == c64 && isDouble) {
-                options << " -D CONVT=double"
-                        << " -D USE_DOUBLE";
-            }
+        addKernelToCache(device, refName, entry);
+    }
 
-            cl::Program prog;
-            buildProgram(prog,
-                          fftconvolve_reorder_cl,
-                          fftconvolve_reorder_cl_len,
-                          options.str());
-            fftconvolveProgs[device] = new Program(prog);
+    int fftScale = 1;
 
-            roKernel[device] = new Kernel(*fftconvolveProgs[device], "reorder_output");
-        });
+    // Calculate the scale by which to divide clFFT results
+    for (int k = 0; k < baseDim; k++)
+        fftScale *= packed.info.dims[k];
 
     Param sig_tmp, filter_tmp;
     calcParamSizes(sig_tmp, filter_tmp, packed, sig, filter, baseDim, kind);
@@ -272,26 +305,18 @@ void reorderOutputHelper(Param out,
     NDRange local(THREADS);
     NDRange global(blocks * THREADS);
 
-    auto roOp = KernelFunctor<Buffer, KParam,
-                            Buffer, KParam,
-                            KParam, const int,
-                            const int, const int> (*roKernel[device]);
+    auto roOp = KernelFunctor< Buffer, KParam, Buffer, KParam, KParam, const int,
+                               const int, const int > (*entry.ker);
 
     if (kind == AF_BATCH_RHS) {
-        roOp(EnqueueArgs(getQueue(), global, local),
-              *out.data, out.info,
-              *filter_tmp.data, filter_tmp.info,
-              filter.info, sig_half_d0, baseDim, fftScale);
-    }
-    else {
-        roOp(EnqueueArgs(getQueue(), global, local),
-              *out.data, out.info,
-              *sig_tmp.data, sig_tmp.info,
-              filter.info, sig_half_d0, baseDim, fftScale);
+        roOp(EnqueueArgs(getQueue(), global, local), *out.data, out.info,
+             *filter_tmp.data, filter_tmp.info, filter.info, sig_half_d0, baseDim, fftScale);
+    } else {
+        roOp(EnqueueArgs(getQueue(), global, local), *out.data, out.info,
+             *sig_tmp.data, sig_tmp.info, filter.info, sig_half_d0, baseDim, fftScale);
     }
+
     CL_DEBUG_FINISH(getQueue());
 }
-
 } // namespace kernel
-
 } // namespace opencl
diff --git a/src/backend/opencl/kernel/gradient.hpp b/src/backend/opencl/kernel/gradient.hpp
index 4dcca9bc94..393fa16743 100644
--- a/src/backend/opencl/kernel/gradient.hpp
+++ b/src/backend/opencl/kernel/gradient.hpp
@@ -12,8 +12,7 @@
 #include <program.hpp>
 #include <traits.hpp>
 #include <string>
-#include <mutex>
-#include <map>
+#include <cache.hpp>
 #include <dispatch.hpp>
 #include <Param.hpp>
 #include <debug_opencl.hpp>
@@ -31,62 +30,64 @@ using std::string;
 
 namespace opencl
 {
-    namespace kernel
-    {
-        // Kernel Launch Config Values
-        static const int TX = 32;
-        static const int TY = 8;
+namespace kernel
+{
+// Kernel Launch Config Values
+static const int TX = 32;
+static const int TY = 8;
 
-        template<typename T>
-        void gradient(Param grad0, Param grad1, const Param in)
-        {
-            static std::once_flag compileFlags[DeviceManager::MAX_DEVICES];
-            static std::map<int, Program*>  gradProgs;
-            static std::map<int, Kernel*> gradKernels;
+template<typename T>
+void gradient(Param grad0, Param grad1, const Param in)
+{
+    std::string refName = std::string("gradient_kernel_") + std::string(dtype_traits<T>::getName());
 
-            int device = getActiveDeviceId();
+    int device = getActiveDeviceId();
+    kc_entry_t entry = kernelCache(device, refName);
 
-            std::call_once( compileFlags[device], [device] () {
-                ToNumStr<T> toNumStr;
-                std::ostringstream options;
-                options << " -D T=" << dtype_traits<T>::getName()
-                        << " -D TX=" << TX
-                        << " -D TY=" << TY
-                        << " -D ZERO=" << toNumStr(scalar<T>(0));
+    if (entry.prog==0 && entry.ker==0) {
+        ToNumStr<T> toNumStr;
+        std::ostringstream options;
+        options << " -D T=" << dtype_traits<T>::getName()
+                << " -D TX=" << TX
+                << " -D TY=" << TY
+                << " -D ZERO=" << toNumStr(scalar<T>(0));
 
-                if((af_dtype) dtype_traits<T>::af_type == c32 ||
-                    (af_dtype) dtype_traits<T>::af_type == c64) {
-                    options << " -D CPLX=1";
-                } else {
-                    options << " -D CPLX=0";
-                }
-                if (std::is_same<T, double>::value ||
-                    std::is_same<T, cdouble>::value) {
-                    options << " -D USE_DOUBLE";
-                }
-                Program prog;
-                buildProgram(prog, gradient_cl, gradient_cl_len, options.str());
-                gradProgs[device]   = new Program(prog);
-                gradKernels[device] = new Kernel(*gradProgs[device], "gradient_kernel");
-            });
+        if((af_dtype) dtype_traits<T>::af_type == c32 ||
+            (af_dtype) dtype_traits<T>::af_type == c64) {
+            options << " -D CPLX=1";
+        } else {
+            options << " -D CPLX=0";
+        }
+        if (std::is_same<T, double>::value ||
+            std::is_same<T, cdouble>::value) {
+            options << " -D USE_DOUBLE";
+        }
 
-            auto gradOp = KernelFunctor<Buffer, const KParam, Buffer, const KParam,
-                                const Buffer, const KParam, const int, const int>
-                                    (*gradKernels[device]);
+        const char* ker_strs[] = {gradient_cl};
+        const int   ker_lens[] = {gradient_cl_len};
+        Program prog;
+        buildProgram(prog, 1, ker_strs, ker_lens, options.str());
+        entry.prog = new Program(prog);
+        entry.ker  = new Kernel(*entry.prog, "gradient_kernel");
 
-            NDRange local(TX, TY, 1);
+        addKernelToCache(device, refName, entry);
+    }
 
-            int blocksPerMatX = divup(in.info.dims[0], TX);
-            int blocksPerMatY = divup(in.info.dims[1], TY);
-            NDRange global(local[0] * blocksPerMatX * in.info.dims[2],
-                            local[1] * blocksPerMatY * in.info.dims[3],
-                            1);
+    auto gradOp = KernelFunctor< Buffer, const KParam, Buffer, const KParam,
+                                 const Buffer, const KParam, const int, const int >(*entry.ker);
 
-            gradOp(EnqueueArgs(getQueue(), global, local),
-                    *grad0.data, grad0.info, *grad1.data, grad1.info,
-                    *in.data, in.info, blocksPerMatX, blocksPerMatY);
+    NDRange local(TX, TY, 1);
 
-            CL_DEBUG_FINISH(getQueue());
-        }
-    }
+    int blocksPerMatX = divup(in.info.dims[0], TX);
+    int blocksPerMatY = divup(in.info.dims[1], TY);
+    NDRange global(local[0] * blocksPerMatX * in.info.dims[2],
+                   local[1] * blocksPerMatY * in.info.dims[3], 1);
+
+    gradOp(EnqueueArgs(getQueue(), global, local),
+           *grad0.data, grad0.info, *grad1.data, grad1.info,
+           *in.data, in.info, blocksPerMatX, blocksPerMatY);
+
+    CL_DEBUG_FINISH(getQueue());
+}
+}
 }
diff --git a/src/backend/opencl/kernel/harris.hpp b/src/backend/opencl/kernel/harris.hpp
index 37785db588..b4cf77decf 100644
--- a/src/backend/opencl/kernel/harris.hpp
+++ b/src/backend/opencl/kernel/harris.hpp
@@ -19,7 +19,8 @@
 #include <kernel/range.hpp>
 #include <kernel_headers/harris.hpp>
 #include <memory.hpp>
-#include <map>
+#include <cache.hpp>
+#include <tuple>
 
 using cl::Buffer;
 using cl::Program;
@@ -30,10 +31,8 @@ using cl::NDRange;
 
 namespace opencl
 {
-
 namespace kernel
 {
-
 static const unsigned HARRIS_THREADS_PER_GROUP = 256;
 static const unsigned HARRIS_THREADS_X = 16;
 static const unsigned HARRIS_THREADS_Y = HARRIS_THREADS_PER_GROUP / HARRIS_THREADS_X;
@@ -85,6 +84,55 @@ void conv_helper(Param &ixx, Param &ixy, Param &iyy, Param &filter)
     bufferFree(iyy_tmp.data);
 }
 
+template<typename T>
+std::tuple<cl::Kernel*, cl::Kernel*, cl::Kernel*, cl::Kernel*>
+getHarrisKernels()
+{
+    static const std::string kernelNames[4] =
+        {"second_order_deriv", "keep_corners", "harris_responses", "non_maximal"};
+
+    kc_entry_t entries[4];
+
+    int device = getActiveDeviceId();
+
+    std::string checkName = kernelNames[0] + std::string("_") + std::string(dtype_traits<T>::getName());
+
+    entries[0] = kernelCache(device, checkName);
+
+    if (entries[0].prog==0 && entries[0].ker==0)
+    {
+        std::ostringstream options;
+        options << " -D T=" << dtype_traits<T>::getName();
+        if (std::is_same<T, double>::value || std::is_same<T, cdouble>::value)
+            options << " -D USE_DOUBLE";
+
+        const char* ker_strs[] = {harris_cl};
+        const int   ker_lens[] = {harris_cl_len};
+        Program prog;
+        buildProgram(prog, 1, ker_strs, ker_lens, options.str());
+
+        for (int i=0; i<4; ++i)
+        {
+            entries[i].prog = new Program(prog);
+            entries[i].ker  = new Kernel(*entries[i].prog, kernelNames[i].c_str());
+
+            std::string name = kernelNames[i] +
+                std::string("_") + std::string(dtype_traits<T>::getName());
+
+            addKernelToCache(device, name, entries[i]);
+        }
+    } else {
+        for (int i=1; i<4; ++i) {
+            std::string name = kernelNames[i] +
+                std::string("_") + std::string(dtype_traits<T>::getName());
+
+            entries[i] = kernelCache(device, name);
+        }
+    }
+
+    return std::make_tuple(entries[0].ker, entries[1].ker, entries[2].ker, entries[3].ker);
+}
+
 template<typename T, typename convAccT>
 void harris(unsigned* corners_out,
             Param &x_out,
@@ -97,34 +145,7 @@ void harris(unsigned* corners_out,
             const unsigned filter_len,
             const float k_thr)
 {
-    static std::once_flag compileFlags[DeviceManager::MAX_DEVICES];
-    static std::map<int, Program*> harrisProgs;
-    static std::map<int, Kernel*>  soKernel;
-    static std::map<int, Kernel*>  kcKernel;
-    static std::map<int, Kernel*>  hrKernel;
-    static std::map<int, Kernel*>  nmKernel;
-
-    int device = getActiveDeviceId();
-
-    std::call_once( compileFlags[device], [device] () {
-
-            std::ostringstream options;
-            options << " -D T=" << dtype_traits<T>::getName();
-
-            if (std::is_same<T, double>::value ||
-                std::is_same<T, cdouble>::value) {
-                options << " -D USE_DOUBLE";
-            }
-
-            cl::Program prog;
-            buildProgram(prog, harris_cl, harris_cl_len, options.str());
-            harrisProgs[device] = new Program(prog);
-
-            soKernel[device] = new Kernel(*harrisProgs[device], "second_order_deriv");
-            kcKernel[device] = new Kernel(*harrisProgs[device], "keep_corners");
-            hrKernel[device] = new Kernel(*harrisProgs[device], "harris_responses");
-            nmKernel[device] = new Kernel(*harrisProgs[device], "non_maximal");
-        });
+    auto kernels = getHarrisKernels<T>();
 
     // Window filter
     convAccT* h_filter = new convAccT[filter_len];
@@ -132,8 +153,7 @@ void harris(unsigned* corners_out,
     if (sigma < 0.5f) {
         for (unsigned i = 0; i < filter_len; i++)
             h_filter[i] = (T)1.f / (filter_len);
-    }
-    else {
+    } else {
         gaussian1D<convAccT>(h_filter, (int)filter_len, sigma);
     }
 
@@ -181,13 +201,13 @@ void harris(unsigned* corners_out,
     const NDRange local_so(HARRIS_THREADS_PER_GROUP, 1);
     const NDRange global_so(blk_x_so * HARRIS_THREADS_PER_GROUP, 1);
 
-    auto soOp = KernelFunctor<Buffer, Buffer, Buffer,
-                            unsigned, Buffer, Buffer> (*soKernel[device]);
+    auto soOp = KernelFunctor< Buffer, Buffer, Buffer,
+                               unsigned, Buffer, Buffer > (*std::get<0>(kernels));
 
     // Compute second-order derivatives
     soOp(EnqueueArgs(getQueue(), global_so, local_so),
-          *ixx.data, *ixy.data, *iyy.data,
-          in.info.dims[3] * in.info.strides[3], *ix.data, *iy.data);
+         *ixx.data, *ixy.data, *iyy.data,
+         in.info.dims[3] * in.info.strides[3], *ix.data, *iy.data);
     CL_DEBUG_FINISH(getQueue());
 
     bufferFree(ix.data);
@@ -205,14 +225,13 @@ void harris(unsigned* corners_out,
     const NDRange local_hr(HARRIS_THREADS_X, HARRIS_THREADS_Y);
     const NDRange global_hr(blk_x_hr * HARRIS_THREADS_X, blk_y_hr * HARRIS_THREADS_Y);
 
-    auto hrOp = KernelFunctor<Buffer, unsigned, unsigned,
-                            Buffer, Buffer, Buffer,
-                            float, unsigned> (*hrKernel[device]);
+    auto hrOp = KernelFunctor< Buffer, unsigned, unsigned, Buffer, Buffer, Buffer,
+                               float, unsigned> (*std::get<2>(kernels));
 
     // Calculate Harris responses for all pixels
     hrOp(EnqueueArgs(getQueue(), global_hr, local_hr),
-          *d_responses, in.info.dims[0], in.info.dims[1],
-          *ixx.data, *ixy.data, *iyy.data, k_thr, border_len);
+         *d_responses, in.info.dims[0], in.info.dims[1],
+         *ixx.data, *ixy.data, *iyy.data, k_thr, border_len);
     CL_DEBUG_FINISH(getQueue());
 
     bufferFree(ixx.data);
@@ -233,15 +252,14 @@ void harris(unsigned* corners_out,
 
     const float min_r = (max_corners > 0) ? 0.f : min_response;
 
-    auto nmOp = KernelFunctor<Buffer, Buffer, Buffer, Buffer,
-                            Buffer, unsigned, unsigned,
-                            float, unsigned, unsigned> (*nmKernel[device]);
+    auto nmOp = KernelFunctor< Buffer, Buffer, Buffer, Buffer, Buffer, unsigned, unsigned,
+                            float, unsigned, unsigned> (*std::get<3>(kernels));
 
     // Perform non-maximal suppression
     nmOp(EnqueueArgs(getQueue(), global_hr, local_hr),
-          *d_x_corners, *d_y_corners, *d_resp_corners, *d_corners_found,
-          *d_responses, in.info.dims[0], in.info.dims[1],
-          min_r, border_len, corner_lim);
+         *d_x_corners, *d_y_corners, *d_resp_corners, *d_corners_found,
+         *d_responses, in.info.dims[0], in.info.dims[1],
+         min_r, border_len, corner_lim);
     CL_DEBUG_FINISH(getQueue());
 
     getQueue().enqueueReadBuffer(*d_corners_found, CL_TRUE, 0, sizeof(unsigned), &corners_found);
@@ -299,16 +317,15 @@ void harris(unsigned* corners_out,
         const NDRange local_kc(HARRIS_THREADS_PER_GROUP, 1);
         const NDRange global_kc(blk_x_kc * HARRIS_THREADS_PER_GROUP, 1);
 
-        auto kcOp = KernelFunctor<Buffer, Buffer, Buffer,
-                                Buffer, Buffer, Buffer, Buffer,
-                                unsigned> (*kcKernel[device]);
+        auto kcOp = KernelFunctor< Buffer, Buffer, Buffer, Buffer, Buffer, Buffer, Buffer,
+                                   unsigned> (*std::get<1>(kernels));
 
         // Keep only the first corners_to_keep corners with higher Harris
         // responses
         kcOp(EnqueueArgs(getQueue(), global_kc, local_kc),
-              *x_out.data, *y_out.data, *resp_out.data,
-              *d_x_corners, *d_y_corners, *harris_resp.data, *harris_idx.data,
-              *corners_out);
+             *x_out.data, *y_out.data, *resp_out.data,
+             *d_x_corners, *d_y_corners, *harris_resp.data, *harris_idx.data,
+             *corners_out);
         CL_DEBUG_FINISH(getQueue());
 
         bufferFree(d_x_corners);
@@ -334,7 +351,5 @@ void harris(unsigned* corners_out,
         resp_out.data = d_resp_corners;
     }
 }
-
 } //namespace kernel
-
 } //namespace opencl
diff --git a/src/backend/opencl/kernel/histogram.hpp b/src/backend/opencl/kernel/histogram.hpp
index 28b0e1ad3d..6a998b6621 100644
--- a/src/backend/opencl/kernel/histogram.hpp
+++ b/src/backend/opencl/kernel/histogram.hpp
@@ -12,8 +12,7 @@
 #include <program.hpp>
 #include <traits.hpp>
 #include <string>
-#include <mutex>
-#include <map>
+#include <cache.hpp>
 #include <dispatch.hpp>
 #include <Param.hpp>
 #include <debug_opencl.hpp>
@@ -23,10 +22,8 @@ using cl::KernelFunctor;
 
 namespace opencl
 {
-
 namespace kernel
 {
-
 static const unsigned MAX_BINS  = 4000;
 static const int THREADS_X =  256;
 static const int THRD_LOAD =   16;
@@ -34,34 +31,38 @@ static const int THRD_LOAD =   16;
 template<typename inType, typename outType, bool isLinear>
 void histogram(Param out, const Param in, int nbins, float minval, float maxval)
 {
-    static std::once_flag compileFlags[DeviceManager::MAX_DEVICES];
-    static std::map<int, Program*> histProgs;
-    static std::map<int, Kernel *> histKernels;
+    std::string refName = std::string("histogram_") +
+        std::string(dtype_traits<inType>::getName()) +
+        std::string(dtype_traits<inType>::getName()) +
+        std::to_string(isLinear);
 
     int device = getActiveDeviceId();
+    kc_entry_t entry = kernelCache(device, refName);
 
-    std::call_once( compileFlags[device], [device] () {
-                std::ostringstream options;
-                options << " -D inType=" << dtype_traits<inType>::getName()
-                        << " -D outType=" << dtype_traits<outType>::getName()
-                        << " -D THRD_LOAD=" << THRD_LOAD;
-                if (isLinear)
-                    options << " -D IS_LINEAR";
-                if (std::is_same<inType, double>::value ||
-                    std::is_same<inType, cdouble>::value) {
-                    options << " -D USE_DOUBLE";
-                }
+    if (entry.prog==0 && entry.ker==0) {
+        std::ostringstream options;
+        options << " -D inType=" << dtype_traits<inType>::getName()
+            << " -D outType=" << dtype_traits<outType>::getName()
+            << " -D THRD_LOAD=" << THRD_LOAD;
+        if (isLinear)
+            options << " -D IS_LINEAR";
+        if (std::is_same<inType, double>::value ||
+                std::is_same<inType, cdouble>::value) {
+            options << " -D USE_DOUBLE";
+        }
 
-                Program prog;
-                buildProgram(prog, histogram_cl, histogram_cl_len, options.str());
-                histProgs[device]   = new Program(prog);
-                histKernels[device] = new Kernel(*histProgs[device], "histogram");
-            });
+        const char* ker_strs[] = {histogram_cl};
+        const int   ker_lens[] = {histogram_cl_len};
+        Program prog;
+        buildProgram(prog, 1, ker_strs, ker_lens, options.str());
+        entry.prog = new Program(prog);
+        entry.ker  = new Kernel(*entry.prog, "histogram");
 
-    auto histogramOp = KernelFunctor<Buffer, KParam, Buffer, KParam,
-                                    cl::LocalSpaceArg,
-                                    int, int, float, float, int
-                                  >(*histKernels[device]);
+        addKernelToCache(device, refName, entry);
+    }
+
+    auto histogramOp = KernelFunctor< Buffer, KParam, Buffer, KParam, cl::LocalSpaceArg,
+                                      int, int, float, float, int >(*entry.ker);
 
     int nElems = in.info.dims[0]*in.info.dims[1];
     int blk_x  = divup(nElems, THRD_LOAD*THREADS_X);
@@ -71,11 +72,10 @@ void histogram(Param out, const Param in, int nbins, float minval, float maxval)
     NDRange global(blk_x*in.info.dims[2]*THREADS_X, in.info.dims[3]);
 
     histogramOp(EnqueueArgs(getQueue(), global, local),
-            *out.data, out.info, *in.data, in.info,
-            cl::Local(locSize), nElems, nbins, minval, maxval, blk_x);
+                *out.data, out.info, *in.data, in.info,
+                cl::Local(locSize), nElems, nbins, minval, maxval, blk_x);
 
     CL_DEBUG_FINISH(getQueue());
 }
-
 }
 }
diff --git a/src/backend/opencl/kernel/homography.hpp b/src/backend/opencl/kernel/homography.hpp
index a5f6faf096..572c21bdca 100644
--- a/src/backend/opencl/kernel/homography.hpp
+++ b/src/backend/opencl/kernel/homography.hpp
@@ -17,6 +17,7 @@
 #include <kernel/reduce.hpp>
 #include <kernel/sort.hpp>
 #include <cfloat>
+#include <cache.hpp>
 
 using cl::Buffer;
 using cl::Program;
@@ -28,68 +29,88 @@ using std::vector;
 
 namespace opencl
 {
-
 namespace kernel
 {
-
 const int HG_THREADS_X = 16;
 const int HG_THREADS_Y = 16;
 const int HG_THREADS   = 256;
 
 template<typename T, af_homography_type htype>
-int computeH(
-    Param bestH,
-    Param H,
-    Param err,
-    Param x_src,
-    Param y_src,
-    Param x_dst,
-    Param y_dst,
-    Param rnd,
-    const unsigned iterations,
-    const unsigned nsamples,
-    const float inlier_thr)
+std::array<cl::Kernel*, 5> getHomographyKernels()
 {
-    static std::once_flag compileFlags[DeviceManager::MAX_DEVICES];
-    static std::map<int, Program*> hgProgs;
-    static std::map<int, Kernel*>  chKernel;
-    static std::map<int, Kernel*>  ehKernel;
-    static std::map<int, Kernel*>  cmKernel;
-    static std::map<int, Kernel*>  fmKernel;
-    static std::map<int, Kernel*>  clKernel;
+    static const unsigned NUM_KERNELS = 5;
+    static const std::string kernelNames[NUM_KERNELS] =
+        {"compute_homography", "eval_homography", "compute_median",
+         "find_min_median", "compute_lmeds_inliers"};
+
+    kc_entry_t entries[NUM_KERNELS];
 
     int device = getActiveDeviceId();
 
-    std::call_once( compileFlags[device], [device] () {
+    std::string checkName = kernelNames[0] + std::string("_") +
+        std::string(dtype_traits<T>::getName()) +
+        std::to_string(htype);
+
+    entries[0] = kernelCache(device, checkName);
 
-            std::ostringstream options;
-            options << " -D T=" << dtype_traits<T>::getName();
+    if (entries[0].prog==0 && entries[0].ker==0)
+    {
+        std::ostringstream options;
+        options << " -D T=" << dtype_traits<T>::getName();
 
-            if (std::is_same<T, double>::value) {
-                options << " -D USE_DOUBLE";
-                options << " -D EPS=" << DBL_EPSILON;
-            } else
-                options << " -D EPS=" << FLT_EPSILON;
+        if (std::is_same<T, double>::value) {
+            options << " -D USE_DOUBLE";
+            options << " -D EPS=" << DBL_EPSILON;
+        } else
+            options << " -D EPS=" << FLT_EPSILON;
+
+        if (htype == AF_HOMOGRAPHY_RANSAC)
+            options << " -D RANSAC";
+        else if (htype == AF_HOMOGRAPHY_LMEDS)
+            options << " -D LMEDS";
+
+        if (getActiveDeviceType() == CL_DEVICE_TYPE_CPU) {
+            options << " -D IS_CPU";
+        }
 
-            if (htype == AF_HOMOGRAPHY_RANSAC)
-                options << " -D RANSAC";
-            else if (htype == AF_HOMOGRAPHY_LMEDS)
-                options << " -D LMEDS";
+        cl::Program prog;
+        buildProgram(prog, homography_cl, homography_cl_len, options.str());
+
+        for (unsigned i=0; i<NUM_KERNELS; ++i)
+        {
+            entries[i].prog = new Program(prog);
+            entries[i].ker  = new Kernel(*entries[i].prog, kernelNames[i].c_str());
+
+            std::string name = kernelNames[i] + std::string("_") +
+                std::string(dtype_traits<T>::getName()) +
+                std::to_string(htype);
+
+            addKernelToCache(device, name, entries[i]);
+        }
+    } else {
+        for (unsigned i=1; i<NUM_KERNELS; ++i) {
+            std::string name = kernelNames[i] + std::string("_") +
+                std::string(dtype_traits<T>::getName()) +
+                std::to_string(htype);
+
+            entries[i] = kernelCache(device, name);
+        }
+    }
 
-            if (getActiveDeviceType() == CL_DEVICE_TYPE_CPU) {
-                options << " -D IS_CPU";
-            }
+    std::array<cl::Kernel*, NUM_KERNELS> retVal;
+    for (unsigned i=0; i<NUM_KERNELS; ++i)
+        retVal[i] = entries[i].ker;
 
-            cl::Program prog;
-            buildProgram(prog, homography_cl, homography_cl_len, options.str());
-            hgProgs[device] = new Program(prog);
+    return retVal;
+}
 
-            chKernel[device] = new Kernel(*hgProgs[device], "compute_homography");
-            ehKernel[device] = new Kernel(*hgProgs[device], "eval_homography");
-            cmKernel[device] = new Kernel(*hgProgs[device], "compute_median");
-            fmKernel[device] = new Kernel(*hgProgs[device], "find_min_median");
-            clKernel[device] = new Kernel(*hgProgs[device], "compute_lmeds_inliers");
-        });
+template<typename T, af_homography_type htype>
+int computeH(Param bestH, Param H, Param err, Param x_src, Param y_src,
+             Param x_dst, Param y_dst, Param rnd, const unsigned iterations,
+             const unsigned nsamples,
+             const float inlier_thr)
+{
+    auto kernels = getHomographyKernels<T, htype>();
 
     const int blk_x_ch = 1;
     const int blk_y_ch = divup(iterations, HG_THREADS_Y);
@@ -97,14 +118,12 @@ int computeH(
     const NDRange global_ch(blk_x_ch * HG_THREADS_X, blk_y_ch * HG_THREADS_Y);
 
     // Build linear system and solve SVD
-    auto chOp = KernelFunctor<Buffer, KParam,
-                            Buffer, Buffer, Buffer, Buffer,
-                            Buffer, KParam, unsigned>(*chKernel[device]);
-
-    chOp(EnqueueArgs(getQueue(), global_ch, local_ch),
-          *H.data, H.info,
-          *x_src.data, *y_src.data, *x_dst.data, *y_dst.data,
-          *rnd.data, rnd.info, iterations);
+    auto chOp = KernelFunctor< Buffer, KParam, Buffer, Buffer, Buffer, Buffer,
+                               Buffer, KParam, unsigned>(*kernels[0]);
+
+    chOp(EnqueueArgs(getQueue(), global_ch, local_ch), *H.data, H.info,
+         *x_src.data, *y_src.data, *x_dst.data, *y_dst.data, *rnd.data, rnd.info, iterations);
+
     CL_DEBUG_FINISH(getQueue());
 
     const int blk_x_eh = divup(iterations, HG_THREADS);
@@ -132,16 +151,14 @@ int computeH(
         median.data = bufferAlloc(sizeof(float));
 
     // Compute (and for RANSAC, evaluate) homographies
-    auto ehOp = KernelFunctor<Buffer, Buffer, Buffer, KParam,
-                            Buffer, KParam,
-                            Buffer, Buffer, Buffer, Buffer,
-                            Buffer, unsigned, unsigned, float>(*ehKernel[device]);
-
-    ehOp(EnqueueArgs(getQueue(), global_eh, local_eh),
-          *inliers.data, *idx.data, *H.data, H.info,
-          *err.data, err.info,
-          *x_src.data, *y_src.data, *x_dst.data, *y_dst.data,
-          *rnd.data, iterations, nsamples, inlier_thr);
+    auto ehOp = KernelFunctor< Buffer, Buffer, Buffer, KParam, Buffer, KParam,
+                               Buffer, Buffer, Buffer, Buffer,
+                               Buffer, unsigned, unsigned, float>(*kernels[1]);
+
+    ehOp(EnqueueArgs(getQueue(), global_eh, local_eh), *inliers.data, *idx.data, *H.data, H.info,
+         *err.data, err.info, *x_src.data, *y_src.data, *x_dst.data, *y_dst.data,
+         *rnd.data, iterations, nsamples, inlier_thr);
+
     CL_DEBUG_FINISH(getQueue());
 
     unsigned inliersH, idxH;
@@ -154,12 +171,11 @@ int computeH(
         float minMedian;
 
         // Compute median of every iteration
-        auto cmOp = KernelFunctor<Buffer, Buffer, Buffer, KParam,
-                                unsigned>(*cmKernel[device]);
+        auto cmOp = KernelFunctor<Buffer, Buffer, Buffer, KParam, unsigned>(*kernels[2]);
 
         cmOp(EnqueueArgs(getQueue(), global_eh, local_eh),
-              *median.data, *idx.data, *err.data, err.info,
-              iterations);
+             *median.data, *idx.data, *err.data, err.info, iterations);
+
         CL_DEBUG_FINISH(getQueue());
 
         // Reduce medians, only in case iterations > 256
@@ -170,12 +186,11 @@ int computeH(
             cl::Buffer* finalMedian = bufferAlloc(sizeof(float));
             cl::Buffer* finalIdx = bufferAlloc(sizeof(unsigned));
 
-            auto fmOp = KernelFunctor<Buffer, Buffer, Buffer, KParam,
-                                    Buffer>(*fmKernel[device]);
+            auto fmOp = KernelFunctor<Buffer, Buffer, Buffer, KParam, Buffer>(*kernels[3]);
 
             fmOp(EnqueueArgs(getQueue(), global_fm, local_fm),
-                  *finalMedian, *finalIdx, *median.data, median.info,
-                  *idx.data);
+                 *finalMedian, *finalIdx, *median.data, median.info, *idx.data);
+
             CL_DEBUG_FINISH(getQueue());
 
             getQueue().enqueueReadBuffer(*finalMedian, CL_TRUE, 0, sizeof(float), &minMedian);
@@ -196,14 +211,12 @@ int computeH(
         const NDRange local_cl(HG_THREADS);
         const NDRange global_cl(blk_x_cl * HG_THREADS);
 
-        auto clOp = KernelFunctor<Buffer, Buffer,
-                                Buffer, Buffer, Buffer, Buffer,
-                                float, unsigned>(*clKernel[device]);
+        auto clOp = KernelFunctor< Buffer, Buffer, Buffer, Buffer, Buffer, Buffer,
+                                   float, unsigned >(*kernels[4]);
+
+        clOp(EnqueueArgs(getQueue(), global_cl, local_cl), *inliers.data, *bestH.data,
+             *x_src.data, *y_src.data, *x_dst.data, *y_dst.data, minMedian, nsamples);
 
-        clOp(EnqueueArgs(getQueue(), global_cl, local_cl),
-              *inliers.data, *bestH.data,
-              *x_src.data, *y_src.data, *x_dst.data, *y_dst.data,
-              minMedian, nsamples);
         CL_DEBUG_FINISH(getQueue());
 
         // Adds up the total number of inliers
@@ -234,7 +247,5 @@ int computeH(
 
     return (int)inliersH;
 }
-
 } // namespace kernel
-
 } // namespace cuda
diff --git a/src/backend/opencl/kernel/hsv_rgb.hpp b/src/backend/opencl/kernel/hsv_rgb.hpp
index 569912c836..0165512173 100644
--- a/src/backend/opencl/kernel/hsv_rgb.hpp
+++ b/src/backend/opencl/kernel/hsv_rgb.hpp
@@ -12,8 +12,7 @@
 #include <program.hpp>
 #include <traits.hpp>
 #include <string>
-#include <mutex>
-#include <map>
+#include <cache.hpp>
 #include <dispatch.hpp>
 #include <Param.hpp>
 #include <debug_opencl.hpp>
@@ -28,37 +27,36 @@ using std::string;
 
 namespace opencl
 {
-
 namespace kernel
 {
-
 static const int THREADS_X = 16;
 static const int THREADS_Y = 16;
 
 template<typename T, bool isHSV2RGB>
 void hsv2rgb_convert(Param out, const Param in)
 {
-    static std::once_flag compileFlags[DeviceManager::MAX_DEVICES];
-    static std::map<int, Program*>  hrProgs;
-    static std::map<int, Kernel*> hrKernels;
+    std::string refName = std::string("hsvrgb_convert_") +
+        std::string(dtype_traits<T>::getName()) + std::to_string(isHSV2RGB);
 
     int device = getActiveDeviceId();
+    kc_entry_t entry = kernelCache(device, refName);
 
-    std::call_once( compileFlags[device], [device] () {
+    if (entry.prog==0 && entry.ker==0) {
+        std::ostringstream options;
+        options << " -D T=" << dtype_traits<T>::getName();
 
-            std::ostringstream options;
-            options << " -D T=" << dtype_traits<T>::getName();
+        if(isHSV2RGB) options << " -D isHSV2RGB";
+        if (std::is_same<T, double>::value) options << " -D USE_DOUBLE";
 
-            if(isHSV2RGB) options << " -D isHSV2RGB";
+        const char* ker_strs[] = {hsv_rgb_cl};
+        const int   ker_lens[] = {hsv_rgb_cl_len};
+        Program prog;
+        buildProgram(prog, 1, ker_strs, ker_lens, options.str());
+        entry.prog = new Program(prog);
+        entry.ker  = new Kernel(*entry.prog, "convert");
 
-            if (std::is_same<T, double>::value) {
-                options << " -D USE_DOUBLE";
-            }
-            Program prog;
-            buildProgram(prog, hsv_rgb_cl, hsv_rgb_cl_len, options.str());
-            hrProgs[device]   = new Program(prog);
-            hrKernels[device] = new Kernel(*hrProgs[device], "convert");
-        });
+        addKernelToCache(device, refName, entry);
+    }
 
     NDRange local(THREADS_X, THREADS_Y);
 
@@ -69,14 +67,11 @@ void hsv2rgb_convert(Param out, const Param in)
     // parameter would be along 4th dimension
     NDRange global(blk_x * in.info.dims[3] * THREADS_X, blk_y * THREADS_Y);
 
-    auto hsvrgbOp = KernelFunctor<Buffer, KParam, Buffer, KParam, int> (*hrKernels[device]);
+    auto hsvrgbOp = KernelFunctor<Buffer, KParam, Buffer, KParam, int> (*entry.ker);
 
-    hsvrgbOp(EnqueueArgs(getQueue(), global, local),
-                *out.data, out.info, *in.data, in.info, blk_x);
+    hsvrgbOp(EnqueueArgs(getQueue(), global, local), *out.data, out.info, *in.data, in.info, blk_x);
 
     CL_DEBUG_FINISH(getQueue());
 }
-
 }
-
 }
diff --git a/src/backend/opencl/kernel/identity.hpp b/src/backend/opencl/kernel/identity.hpp
index 3882de037c..49197ba2d3 100644
--- a/src/backend/opencl/kernel/identity.hpp
+++ b/src/backend/opencl/kernel/identity.hpp
@@ -13,8 +13,7 @@
 #include <dispatch.hpp>
 #include <Param.hpp>
 #include <debug_opencl.hpp>
-#include <map>
-#include <mutex>
+#include <cache.hpp>
 #include <math.hpp>
 #include "config.hpp"
 
@@ -30,49 +29,46 @@ using af::scalar_to_option;
 
 namespace opencl
 {
-
 namespace kernel
 {
+template<typename T>
+static void identity(Param out)
+{
+    std::string refName = std::string("identity_kernel") + std::string(dtype_traits<T>::getName());
 
-    template<typename T>
-    static void identity(Param out)
-    {
-        static std::once_flag compileFlags[DeviceManager::MAX_DEVICES];
-        static std::map<int, Program*>   identityProgs;
-        static std::map<int, Kernel*>  identityKernels;
+    int device = getActiveDeviceId();
+    kc_entry_t entry = kernelCache(device, refName);
 
-        int device = getActiveDeviceId();
+    if (entry.prog==0 && entry.ker==0) {
+        std::ostringstream options;
+        options << " -D T="    << dtype_traits<T>::getName()
+            << " -D ONE=(T)("  << scalar_to_option(scalar<T>(1)) << ")"
+            << " -D ZERO=(T)(" << scalar_to_option(scalar<T>(0)) << ")";
+        if (std::is_same<T, double>::value ||
+                std::is_same<T, cdouble>::value) {
+            options << " -D USE_DOUBLE";
+        }
 
-        std::call_once( compileFlags[device], [device] () {
-                ostringstream options;
-                options << " -D T="    << dtype_traits<T>::getName()
-                        << " -D ONE=(T)("  << scalar_to_option(scalar<T>(1)) << ")"
-                        << " -D ZERO=(T)(" << scalar_to_option(scalar<T>(0)) << ")";
-                if (std::is_same<T, double>::value ||
-                    std::is_same<T, cdouble>::value) {
-                    options << " -D USE_DOUBLE";
-                }
-                Program prog;
-                buildProgram(prog, identity_cl, identity_cl_len, options.str());
-                identityProgs[device]   = new Program(prog);
-                identityKernels[device] = new Kernel(*identityProgs[device], "identity_kernel");
-            });
+        const char* ker_strs[] = {identity_cl};
+        const int   ker_lens[] = {identity_cl_len};
+        Program prog;
+        buildProgram(prog, 1, ker_strs, ker_lens, options.str());
+        entry.prog = new Program(prog);
+        entry.ker  = new Kernel(*entry.prog, "identity_kernel");
 
-        NDRange local(32, 8);
-        int groups_x = divup(out.info.dims[0], local[0]);
-        int groups_y = divup(out.info.dims[1], local[1]);
-        NDRange global(groups_x * out.info.dims[2] * local[0],
-                        groups_y * out.info.dims[3] * local[1]);
+        addKernelToCache(device, refName, entry);
+    }
 
-        auto identityOp = KernelFunctor<Buffer, const KParam,
-                                      int, int> (*identityKernels[device]);
+    NDRange local(32, 8);
+    int groups_x = divup(out.info.dims[0], local[0]);
+    int groups_y = divup(out.info.dims[1], local[1]);
+    NDRange global(groups_x * out.info.dims[2] * local[0], groups_y * out.info.dims[3] * local[1]);
 
-        identityOp(EnqueueArgs(getQueue(), global, local),
-                    *(out.data), out.info, groups_x, groups_y);
-        CL_DEBUG_FINISH(getQueue());
+    auto identityOp = KernelFunctor<Buffer, const KParam, int, int> (*entry.ker);
 
-    }
+    identityOp(EnqueueArgs(getQueue(), global, local), *(out.data), out.info, groups_x, groups_y);
 
+    CL_DEBUG_FINISH(getQueue());
+}
 }
-
 }
diff --git a/src/backend/opencl/kernel/iir.hpp b/src/backend/opencl/kernel/iir.hpp
index d9b220195b..ef3025e7f6 100644
--- a/src/backend/opencl/kernel/iir.hpp
+++ b/src/backend/opencl/kernel/iir.hpp
@@ -12,8 +12,7 @@
 #include <program.hpp>
 #include <traits.hpp>
 #include <string>
-#include <mutex>
-#include <map>
+#include <cache.hpp>
 #include <dispatch.hpp>
 #include <Param.hpp>
 #include <debug_opencl.hpp>
@@ -30,69 +29,59 @@ using af::scalar_to_option;
 
 namespace opencl
 {
+namespace kernel
+{
+template<typename T, bool batch_a>
+void iir(Param y, Param c, Param a)
+{
+    //FIXME: This is a temporary fix. Ideally the local memory should be allocted outside
+    static const int MAX_A_SIZE = (1024 * sizeof(double)) / sizeof(T);
 
-    namespace kernel
-    {
-        template<typename T, bool batch_a>
-        void iir(Param y, Param c, Param a)
-        {
-
-            //FIXME: This is a temporary fix. Ideally the local memory should be allocted outside
-            static const int MAX_A_SIZE = (1024 * sizeof(double)) / sizeof(T);
-
-            static std::once_flag compileFlags[DeviceManager::MAX_DEVICES];
-            static std::map<int, Program*>  iirProgs;
-            static std::map<int, Kernel*> iirKernels;
-
-            int device = getActiveDeviceId();
-
-            std::call_once(compileFlags[device], [device] () {
-
-                    std::ostringstream options;
-                    options << " -D MAX_A_SIZE=" << MAX_A_SIZE
-                            << " -D BATCH_A=" << batch_a
-                            << " -D ZERO=(T)(" << scalar_to_option(scalar<T>(0)) << ")"
-                            << " -D T=" << dtype_traits<T>::getName();
-
-                    if (std::is_same<T, double>::value ||
-                        std::is_same<T, cdouble>::value) {
-                        options << " -D USE_DOUBLE";
-                    }
-
-                    cl::Program prog;
-                    buildProgram(prog, iir_cl, iir_cl_len, options.str());
-                    iirProgs[device] = new Program(prog);
+    std::string refName = std::string("iir_kernel_") +
+        std::string(dtype_traits<T>::getName()) + std::to_string(batch_a);
 
-                    iirKernels[device] = new Kernel(*iirProgs[device], "iir_kernel");
-                });
+    int device = getActiveDeviceId();
+    kc_entry_t entry = kernelCache(device, refName);
 
+    if (entry.prog==0 && entry.ker==0) {
+        std::ostringstream options;
+        options << " -D MAX_A_SIZE=" << MAX_A_SIZE
+            << " -D BATCH_A=" << batch_a
+            << " -D ZERO=(T)(" << scalar_to_option(scalar<T>(0)) << ")"
+            << " -D T=" << dtype_traits<T>::getName();
 
-            const int groups_y = y.info.dims[1];
-            const int groups_x = y.info.dims[2];
+        if (std::is_same<T, double>::value || std::is_same<T, cdouble>::value)
+            options << " -D USE_DOUBLE";
 
-            int threads = 256;
-            while (threads > (int)y.info.dims[0] && threads > 32) threads /= 2;
+        const char* ker_strs[] = {iir_cl};
+        const int   ker_lens[] = {iir_cl_len};
+        Program prog;
+        buildProgram(prog, 1, ker_strs, ker_lens, options.str());
+        entry.prog = new Program(prog);
+        entry.ker  = new Kernel(*entry.prog, "iir_kernel");
 
+        addKernelToCache(device, refName, entry);
+    }
 
-            NDRange local(threads, 1);
-            NDRange global(groups_x * local[0],
-                           groups_y * y.info.dims[3] * local[1]);
+    const int groups_y = y.info.dims[1];
+    const int groups_x = y.info.dims[2];
 
-            auto iirOp = KernelFunctor<Buffer, KParam,
-                                     Buffer, KParam,
-                                     Buffer, KParam,
-                                     int>(*iirKernels[device]);
+    int threads = 256;
+    while (threads > (int)y.info.dims[0] && threads > 32) threads /= 2;
 
-            try {
-                iirOp(EnqueueArgs(getQueue(), global, local),
-                      *y.data, y.info, *c.data, c.info, *a.data, a.info, groups_y);
-            } catch(cl::Error &clerr) {
-                AF_ERROR("Size of a too big for this datatype",
-                         AF_ERR_SIZE);
-            }
+    NDRange local(threads, 1);
+    NDRange global(groups_x * local[0], groups_y * y.info.dims[3] * local[1]);
 
-            CL_DEBUG_FINISH(getQueue());
-        }
+    auto iirOp = KernelFunctor<Buffer, KParam, Buffer, KParam, Buffer, KParam, int>(*entry.ker);
 
+    try {
+        iirOp(EnqueueArgs(getQueue(), global, local),
+              *y.data, y.info, *c.data, c.info, *a.data, a.info, groups_y);
+    } catch(cl::Error &clerr) {
+        AF_ERROR("Size of a too big for this datatype", AF_ERR_SIZE);
     }
+
+    CL_DEBUG_FINISH(getQueue());
+}
+}
 }
diff --git a/src/backend/opencl/kernel/index.hpp b/src/backend/opencl/kernel/index.hpp
index ae39e1f64a..4b41c82172 100644
--- a/src/backend/opencl/kernel/index.hpp
+++ b/src/backend/opencl/kernel/index.hpp
@@ -12,8 +12,7 @@
 #include <program.hpp>
 #include <traits.hpp>
 #include <string>
-#include <mutex>
-#include <map>
+#include <cache.hpp>
 #include <dispatch.hpp>
 #include <Param.hpp>
 #include <debug_opencl.hpp>
@@ -28,10 +27,8 @@ using std::string;
 
 namespace opencl
 {
-
 namespace kernel
 {
-
 static const int THREADS_X = 32;
 static const int THREADS_Y =  8;
 
@@ -44,37 +41,37 @@ typedef struct {
 template<typename T>
 void index(Param out, const Param in, const IndexKernelParam_t& p, Buffer *bPtr[4])
 {
-    static std::once_flag compileFlags[DeviceManager::MAX_DEVICES];
-    static std::map<int, Program*>  idxProgs;
-    static std::map<int, Kernel*> idxKernels;
+    std::string refName = std::string("indexKernel_") + std::string(dtype_traits<T>::getName());
 
     int device = getActiveDeviceId();
+    kc_entry_t entry = kernelCache(device, refName);
 
-    std::call_once( compileFlags[device], [device] () {
-            std::ostringstream options;
-            options << " -D T=" << dtype_traits<T>::getName();
+    if (entry.prog==0 && entry.ker==0) {
+        std::ostringstream options;
 
-            if (std::is_same<T, double>::value ||
-                std::is_same<T, cdouble>::value) {
+        options << " -D T=" << dtype_traits<T>::getName();
+        if (std::is_same<T, double>::value || std::is_same<T, cdouble>::value)
             options << " -D USE_DOUBLE";
-            }
 
-            Program prog;
-            buildProgram(prog, index_cl, index_cl_len, options.str());
-            idxProgs[device]   = new Program(prog);
-            idxKernels[device] = new Kernel(*idxProgs[device], "indexKernel");
-            });
+        const char* ker_strs[] = {index_cl};
+        const int   ker_lens[] = {index_cl_len};
+        Program prog;
+        buildProgram(prog, 1, ker_strs, ker_lens, options.str());
+        entry.prog = new Program(prog);
+        entry.ker  = new Kernel(*entry.prog, "indexKernel");
+
+        addKernelToCache(device, refName, entry);
+    }
 
     NDRange local(THREADS_X, THREADS_Y);
 
     int blk_x = divup(out.info.dims[0], THREADS_X);
     int blk_y = divup(out.info.dims[1], THREADS_Y);
 
-    NDRange global(blk_x * out.info.dims[2] * THREADS_X,
-            blk_y * out.info.dims[3] * THREADS_Y);
+    NDRange global(blk_x * out.info.dims[2] * THREADS_X, blk_y * out.info.dims[3] * THREADS_Y);
 
     auto indexOp = KernelFunctor<Buffer, KParam, Buffer, KParam, IndexKernelParam_t,
-          Buffer, Buffer, Buffer, Buffer, int, int>(*idxKernels[device]);
+                                 Buffer, Buffer, Buffer, Buffer, int, int>(*entry.ker);
 
     indexOp(EnqueueArgs(getQueue(), global, local),
             *out.data, out.info, *in.data, in.info, p,
@@ -82,7 +79,5 @@ void index(Param out, const Param in, const IndexKernelParam_t& p, Buffer *bPtr[
 
     CL_DEBUG_FINISH(getQueue());
 }
-
 }
-
 }
diff --git a/src/backend/opencl/kernel/iota.hpp b/src/backend/opencl/kernel/iota.hpp
index d018357ab2..8e0d85e41e 100644
--- a/src/backend/opencl/kernel/iota.hpp
+++ b/src/backend/opencl/kernel/iota.hpp
@@ -13,8 +13,7 @@
 #include <program.hpp>
 #include <traits.hpp>
 #include <string>
-#include <mutex>
-#include <map>
+#include <cache.hpp>
 #include <dispatch.hpp>
 #include <Param.hpp>
 #include <debug_opencl.hpp>
@@ -29,54 +28,56 @@ using std::string;
 
 namespace opencl
 {
-    namespace kernel
-    {
-        // Kernel Launch Config Values
-        static const int IOTA_TX = 32;
-        static const int IOTA_TY = 8;
-        static const int TILEX = 512;
-        static const int TILEY = 32;
+namespace kernel
+{
+// Kernel Launch Config Values
+static const int IOTA_TX = 32;
+static const int IOTA_TY = 8;
+static const int TILEX = 512;
+static const int TILEY = 32;
 
-        template<typename T>
-        void iota(Param out, const af::dim4 &sdims, const af::dim4 &tdims)
-        {
-            static std::once_flag compileFlags[DeviceManager::MAX_DEVICES];
-            static std::map<int, Program*>  iotaProgs;
-            static std::map<int, Kernel*> iotaKernels;
+template<typename T>
+void iota(Param out, const af::dim4 &sdims, const af::dim4 &tdims)
+{
+    std::string refName = std::string("iota_kernel_") + std::string(dtype_traits<T>::getName());
 
-            int device = getActiveDeviceId();
+    int device = getActiveDeviceId();
+    kc_entry_t entry = kernelCache(device, refName);
 
-            std::call_once( compileFlags[device], [device] () {
-                std::ostringstream options;
-                options << " -D T=" << dtype_traits<T>::getName();
-                if (std::is_same<T, double>::value ||
-                    std::is_same<T, cdouble>::value) {
-                    options << " -D USE_DOUBLE";
-                }
-                Program prog;
-                buildProgram(prog, iota_cl, iota_cl_len, options.str());
-                iotaProgs[device]   = new Program(prog);
-                iotaKernels[device] = new Kernel(*iotaProgs[device], "iota_kernel");
-            });
+    if (entry.prog==0 && entry.ker==0) {
+        std::ostringstream options;
 
-            auto iotaOp = KernelFunctor<Buffer, const KParam,
-                                      const int, const int, const int, const int,
-                                      const int, const int, const int, const int,
-                                      const int, const int> (*iotaKernels[device]);
+        options << " -D T=" << dtype_traits<T>::getName();
+        if (std::is_same<T, double>::value || std::is_same<T, cdouble>::value)
+            options << " -D USE_DOUBLE";
 
-            NDRange local(IOTA_TX, IOTA_TY, 1);
+        const char* ker_strs[] = {iota_cl};
+        const int   ker_lens[] = {iota_cl_len};
+        Program prog;
+        buildProgram(prog, 1, ker_strs, ker_lens, options.str());
+        entry.prog = new Program(prog);
+        entry.ker  = new Kernel(*entry.prog, "iota_kernel");
 
-            int blocksPerMatX = divup(out.info.dims[0], TILEX);
-            int blocksPerMatY = divup(out.info.dims[1], TILEY);
-            NDRange global(local[0] * blocksPerMatX * out.info.dims[2],
-                            local[1] * blocksPerMatY * out.info.dims[3],
-                            1);
+        addKernelToCache(device, refName, entry);
+    }
 
-            iotaOp(EnqueueArgs(getQueue(), global, local),
-                    *out.data, out.info, sdims[0], sdims[1], sdims[2], sdims[3],
-                    tdims[0], tdims[1], tdims[2], tdims[3], blocksPerMatX, blocksPerMatY);
+    auto iotaOp = KernelFunctor<Buffer, const KParam,
+                                const int, const int, const int, const int,
+                                const int, const int, const int, const int,
+                                const int, const int> (*entry.ker);
 
-            CL_DEBUG_FINISH(getQueue());
-        }
-    }
+    NDRange local(IOTA_TX, IOTA_TY, 1);
+
+    int blocksPerMatX = divup(out.info.dims[0], TILEX);
+    int blocksPerMatY = divup(out.info.dims[1], TILEY);
+    NDRange global(local[0] * blocksPerMatX * out.info.dims[2],
+                   local[1] * blocksPerMatY * out.info.dims[3], 1);
+
+    iotaOp(EnqueueArgs(getQueue(), global, local),
+           *out.data, out.info, sdims[0], sdims[1], sdims[2], sdims[3],
+           tdims[0], tdims[1], tdims[2], tdims[3], blocksPerMatX, blocksPerMatY);
+
+    CL_DEBUG_FINISH(getQueue());
+}
+}
 }
diff --git a/src/backend/opencl/kernel/join.hpp b/src/backend/opencl/kernel/join.hpp
index 52cfcaeb98..a450f22fcf 100644
--- a/src/backend/opencl/kernel/join.hpp
+++ b/src/backend/opencl/kernel/join.hpp
@@ -28,59 +28,62 @@ using std::string;
 
 namespace opencl
 {
-    namespace kernel
-    {
-        // Kernel Launch Config Values
-        static const int TX = 32;
-        static const int TY = 8;
-        static const int TILEX = 256;
-        static const int TILEY = 32;
+namespace kernel
+{
+// Kernel Launch Config Values
+static const int TX = 32;
+static const int TY = 8;
+static const int TILEX = 256;
+static const int TILEY = 32;
 
-        template<typename To, typename Ti, int dim>
-        void join(Param out, const Param in, const af::dim4 offset)
-        {
-            static std::once_flag compileFlags[DeviceManager::MAX_DEVICES];
-            static std::map<int, Program*>   joinProgs;
-            static std::map<int, Kernel *> joinKernels;
+template<typename To, typename Ti, int dim>
+void join(Param out, const Param in, const af::dim4 offset)
+{
+    std::string refName = std::string("join_kernel_") +
+        std::string(dtype_traits<To>::getName()) +
+        std::string(dtype_traits<Ti>::getName()) +
+        std::to_string(dim);
 
-            int device = getActiveDeviceId();
+    int device = getActiveDeviceId();
+    kc_entry_t entry = kernelCache(device, refName);
 
-            std::call_once( compileFlags[device], [device] () {
-                std::ostringstream options;
-                options << " -D To=" << dtype_traits<To>::getName()
-                        << " -D Ti=" << dtype_traits<Ti>::getName()
-                        << " -D dim=" << dim;
+    if (entry.prog==0 && entry.ker==0) {
+        std::ostringstream options;
+        options << " -D To=" << dtype_traits<To>::getName()
+                << " -D Ti=" << dtype_traits<Ti>::getName()
+                << " -D dim=" << dim;
 
-                if (std::is_same<To, double>::value ||
-                    std::is_same<To, cdouble>::value) {
-                    options << " -D USE_DOUBLE";
-                } else if (std::is_same<Ti, double>::value ||
-                            std::is_same<Ti, cdouble>::value) {
-                    options << " -D USE_DOUBLE";
-                }
+        if (std::is_same<To, double>::value || std::is_same<To, cdouble>::value) {
+            options << " -D USE_DOUBLE";
+        } else if (std::is_same<Ti, double>::value || std::is_same<Ti, cdouble>::value) {
+            options << " -D USE_DOUBLE";
+        }
 
-                Program prog;
-                buildProgram(prog, join_cl, join_cl_len, options.str());
-                joinProgs[device] = new Program(prog);
-                joinKernels[device] = new Kernel(*joinProgs[device], "join_kernel");
-            });
+        const char* ker_strs[] = {join_cl};
+        const int   ker_lens[] = {join_cl_len};
+        Program prog;
+        buildProgram(prog, 1, ker_strs, ker_lens, options.str());
+        entry.prog = new Program(prog);
+        entry.ker  = new Kernel(*entry.prog, "join_kernel");
 
-            auto joinOp = KernelFunctor<Buffer, const KParam, const Buffer, const KParam,
-                          const int, const int, const int, const int,
-                          const int, const int> (*joinKernels[device]);
+        addKernelToCache(device, refName, entry);
+    }
 
-            NDRange local(TX, TY, 1);
+    auto joinOp = KernelFunctor<Buffer, const KParam, const Buffer, const KParam,
+                                const int, const int, const int, const int,
+                                const int, const int> (*entry.ker);
 
-            int blocksPerMatX = divup(in.info.dims[0], TILEX);
-            int blocksPerMatY = divup(in.info.dims[1], TILEY);
-            NDRange global(local[0] * blocksPerMatX * in.info.dims[2],
-                            local[1] * blocksPerMatY * in.info.dims[3],
-                            1);
+    NDRange local(TX, TY, 1);
 
-            joinOp(EnqueueArgs(getQueue(), global, local), *out.data, out.info, *in.data, in.info,
-                    offset[0], offset[1], offset[2], offset[3], blocksPerMatX, blocksPerMatY);
+    int blocksPerMatX = divup(in.info.dims[0], TILEX);
+    int blocksPerMatY = divup(in.info.dims[1], TILEY);
+    NDRange global(local[0] * blocksPerMatX * in.info.dims[2],
+                   local[1] * blocksPerMatY * in.info.dims[3], 1);
 
-            CL_DEBUG_FINISH(getQueue());
-        }
-    }
+    joinOp(EnqueueArgs(getQueue(), global, local), *out.data, out.info, *in.data, in.info,
+           offset[0], offset[1], offset[2], offset[3], blocksPerMatX, blocksPerMatY);
+
+    CL_DEBUG_FINISH(getQueue());
+}
+}
 }
diff --git a/src/backend/opencl/kernel/laset.hpp b/src/backend/opencl/kernel/laset.hpp
index 8f5fc1f432..612df24383 100644
--- a/src/backend/opencl/kernel/laset.hpp
+++ b/src/backend/opencl/kernel/laset.hpp
@@ -12,8 +12,7 @@
 #include <program.hpp>
 #include <traits.hpp>
 #include <string>
-#include <mutex>
-#include <map>
+#include <cache.hpp>
 #include <dispatch.hpp>
 #include <Param.hpp>
 #include <debug_opencl.hpp>
@@ -28,13 +27,10 @@ using cl::EnqueueArgs;
 using cl::NDRange;
 using std::string;
 
-
 namespace opencl
 {
-
 namespace kernel
 {
-
 static const int BLK_X = 64;
 static const int BLK_Y = 32;
 
@@ -49,46 +45,45 @@ void laset(int m, int  n,
            T offdiag, T diag,
            cl_mem dA, size_t dA_offset, magma_int_t ldda)
 {
-
-    static std::once_flag compileFlags[DeviceManager::MAX_DEVICES];
-    static std::map<int, Program*>  setProgs;
-    static std::map<int, Kernel*> setKernels;
+    std::string refName = laset_name<uplo>() + std::string("_") +
+        std::string(dtype_traits<T>::getName()) +
+        std::to_string(uplo);
 
     int device = getActiveDeviceId();
+    kc_entry_t entry = kernelCache(device, refName);
 
-    std::call_once(compileFlags[device], [device] () {
+    if (entry.prog==0 && entry.ker==0) {
+        std::ostringstream options;
+        options << " -D T=" << dtype_traits<T>::getName()
+                << " -D BLK_X=" << BLK_X
+                << " -D BLK_Y=" << BLK_Y
+                << " -D IS_CPLX=" << af::iscplx<T>();
 
-            std::ostringstream options;
-            options << " -D T=" << dtype_traits<T>::getName()
-                    << " -D BLK_X=" << BLK_X
-                    << " -D BLK_Y=" << BLK_Y
-                    << " -D IS_CPLX=" << af::iscplx<T>();
+        if (std::is_same<T, double>::value || std::is_same<T, cdouble>::value)
+            options << " -D USE_DOUBLE";
 
-            if (std::is_same<T, double>::value ||
-                std::is_same<T, cdouble>::value) {
-                options << " -D USE_DOUBLE";
-            }
+        const char* ker_strs[] = {laset_cl};
+        const int   ker_lens[] = {laset_cl_len};
+        Program prog;
+        buildProgram(prog, 1, ker_strs, ker_lens, options.str());
+        entry.prog = new Program(prog);
+        entry.ker  = new Kernel(*entry.prog, laset_name<uplo>());
 
-            cl::Program prog;
-            buildProgram(prog, laset_cl, laset_cl_len, options.str());
-            setProgs[device] = new Program(prog);
-            setKernels[device] = new Kernel(*setProgs[device], laset_name<uplo>());
-        });
+        addKernelToCache(device, refName, entry);
+    }
 
     int groups_x = (m - 1) / BLK_X + 1;
     int groups_y = (n - 1) / BLK_Y + 1;
 
     NDRange local(BLK_X, 1);
-    NDRange global(groups_x * local[0],
-                   groups_y * local[1]);
+    NDRange global(groups_x * local[0], groups_y * local[1]);
 
     // retain the cl_mem object during cl::Buffer creation
     cl::Buffer dAObj(dA, true);
 
-    auto lasetOp = KernelFunctor<int, int, T, T, Buffer, unsigned long long, int>(*setKernels[device]);
-    lasetOp(EnqueueArgs(getQueue(), global, local),
-            m, n, offdiag, diag, dAObj, dA_offset, ldda);
-}
+    auto lasetOp = KernelFunctor<int, int, T, T, Buffer, unsigned long long, int>(*entry.ker);
 
+    lasetOp(EnqueueArgs(getQueue(), global, local), m, n, offdiag, diag, dAObj, dA_offset, ldda);
+}
 }
 }
diff --git a/src/backend/opencl/kernel/laset_band.hpp b/src/backend/opencl/kernel/laset_band.hpp
index 915be6f560..645622c279 100644
--- a/src/backend/opencl/kernel/laset_band.hpp
+++ b/src/backend/opencl/kernel/laset_band.hpp
@@ -12,8 +12,7 @@
 #include <program.hpp>
 #include <traits.hpp>
 #include <string>
-#include <mutex>
-#include <map>
+#include <cache.hpp>
 #include <dispatch.hpp>
 #include <Param.hpp>
 #include <debug_opencl.hpp>
@@ -28,13 +27,10 @@ using cl::EnqueueArgs;
 using cl::NDRange;
 using std::string;
 
-
 namespace opencl
 {
-
 namespace kernel
 {
-
 #if 0 // Needs to be enabled when unmqr2 is enabled
 static const int NB = 64;
 template<int num>
@@ -47,30 +43,31 @@ void laset_band(int m, int  n, int k,
                 T offdiag, T diag,
                 cl_mem dA, size_t dA_offset, magma_int_t ldda)
 {
-
-    static std::once_flag compileFlags[DeviceManager::MAX_DEVICES];
-    static std::map<int, Program*>  setProgs;
-    static std::map<int, Kernel*> setKernels;
+    std::string refName = laset_band_name<uplo>() + std::string("_") +
+        std::string(dtype_traits<T>::getName()) +
+        std::to_string(uplo);
 
     int device = getActiveDeviceId();
+    kc_entry_t entry = kernelCache(device, refName);
 
-    std::call_once(compileFlags[device], [device] () {
+    if (entry.prog==0 && entry.ker==0) {
+        std::ostringstream options;
+        options << " -D T=" << dtype_traits<T>::getName()
+            << " -D NB=" << NB
+            << " -D IS_CPLX=" << af::iscplx<T>();
 
-            std::ostringstream options;
-            options << " -D T=" << dtype_traits<T>::getName()
-                    << " -D NB=" << NB
-                    << " -D IS_CPLX=" << af::iscplx<T>();
+        if (std::is_same<T, double>::value || std::is_same<T, cdouble>::value)
+            options << " -D USE_DOUBLE";
 
-            if (std::is_same<T, double>::value ||
-                std::is_same<T, cdouble>::value) {
-                options << " -D USE_DOUBLE";
-            }
+        const char* ker_strs[] = {laset_band_cl};
+        const int   ker_lens[] = {laset_band_cl_len};
+        Program prog;
+        buildProgram(prog, 1, ker_strs, ker_lens, options.str());
+        entry.prog = new Program(prog);
+        entry.ker  = new Kernel(*entry.prog, laset_band_name<uplo>());
 
-            cl::Program prog;
-            buildProgram(prog, laset_band_cl, laset_band_cl_len, options.str());
-            setProgs[device] = new Program(prog);
-            setKernels[device] = new Kernel(*setProgs[device], laset_band_name<uplo>());
-        });
+        addKernelToCache(device, refName, entry);
+    }
 
     int threads = 1;
     int groups = 1;
@@ -86,12 +83,10 @@ void laset_band(int m, int  n, int k,
     NDRange local(threads, 1);
     NDRange global(threads * groups, 1);
 
-    auto lasetBandOp = KernelFunctor<int, int, T, T, cl_mem, unsigned long long, int>(*setKernels[device]);
+    auto lasetBandOp = KernelFunctor<int, int, T, T, cl_mem, unsigned long long, int>(*entry.ker);
 
-    lasetBandOp(EnqueueArgs(getQueue(), global, local),
-                m, n, offdiag, diag, dA, dA_offset, ldda);
+    lasetBandOp(EnqueueArgs(getQueue(), global, local), m, n, offdiag, diag, dA, dA_offset, ldda);
 }
 #endif
-
 }
 }
diff --git a/src/backend/opencl/kernel/laswp.hpp b/src/backend/opencl/kernel/laswp.hpp
index 99eb4096ed..9d6b486c3b 100644
--- a/src/backend/opencl/kernel/laswp.hpp
+++ b/src/backend/opencl/kernel/laswp.hpp
@@ -12,8 +12,7 @@
 #include <program.hpp>
 #include <traits.hpp>
 #include <string>
-#include <mutex>
-#include <map>
+#include <cache.hpp>
 #include <dispatch.hpp>
 #include <Param.hpp>
 #include <debug_opencl.hpp>
@@ -27,13 +26,10 @@ using cl::EnqueueArgs;
 using cl::NDRange;
 using std::string;
 
-
 namespace opencl
 {
-
 namespace kernel
 {
-
 static const int NTHREADS = 256;
 static const int MAX_PIVOTS =  32;
 
@@ -42,35 +38,31 @@ typedef struct {
     int ipiv[MAX_PIVOTS];
 } zlaswp_params_t;
 
-
 template<typename T>
-void laswp(int n, cl_mem in, size_t offset, int ldda,
-           int k1, int k2, const int *ipiv, int inci)
+void laswp(int n, cl_mem in, size_t offset, int ldda, int k1, int k2, const int *ipiv, int inci)
 {
-
-    static std::once_flag compileFlags[DeviceManager::MAX_DEVICES];
-    static std::map<int, Program*>  swpProgs;
-    static std::map<int, Kernel*> swpKernels;
+    std::string refName = std::string("laswp_") + std::string(dtype_traits<T>::getName());
 
     int device = getActiveDeviceId();
+    kc_entry_t entry = kernelCache(device, refName);
 
-    std::call_once(compileFlags[device], [device] () {
+    if (entry.prog==0 && entry.ker==0) {
+        std::ostringstream options;
+        options << " -D T=" << dtype_traits<T>::getName()
+                << " -D MAX_PIVOTS=" << MAX_PIVOTS;
 
-            std::ostringstream options;
-            options << " -D T=" << dtype_traits<T>::getName()
-                    << " -D MAX_PIVOTS=" << MAX_PIVOTS;
+        if (std::is_same<T, double>::value || std::is_same<T, cdouble>::value)
+            options << " -D USE_DOUBLE";
 
-            if (std::is_same<T, double>::value ||
-                std::is_same<T, cdouble>::value) {
-                options << " -D USE_DOUBLE";
-            }
+        const char* ker_strs[] = {laswp_cl};
+        const int   ker_lens[] = {laswp_cl_len};
+        Program prog;
+        buildProgram(prog, 1, ker_strs, ker_lens, options.str());
+        entry.prog = new Program(prog);
+        entry.ker  = new Kernel(*entry.prog, "laswp");
 
-            cl::Program prog;
-            buildProgram(prog, laswp_cl, laswp_cl_len, options.str());
-            swpProgs[device] = new Program(prog);
-
-            swpKernels[device] = new Kernel(*swpProgs[device], "laswp");
-        });
+        addKernelToCache(device, refName, entry);
+    }
 
     int groups = divup(n, NTHREADS);
     NDRange local(NTHREADS);
@@ -80,26 +72,20 @@ void laswp(int n, cl_mem in, size_t offset, int ldda,
     //retain the cl_mem object during cl::Buffer creation
     cl::Buffer inObj(in, true);
 
-    auto laswpOp = KernelFunctor<int, Buffer, unsigned long long,
-                               int, zlaswp_params_t>(*swpKernels[device]);
+    auto laswpOp = KernelFunctor<int, Buffer, unsigned long long, int, zlaswp_params_t>(*entry.ker);
 
     for( int k = k1-1; k < k2; k += MAX_PIVOTS ) {
-
         int pivots_left = k2-k;
 
         params.npivots = pivots_left > MAX_PIVOTS ? MAX_PIVOTS : pivots_left;
 
-        for( int j = 0; j < params.npivots; ++j ) {
+        for( int j = 0; j < params.npivots; ++j )
             params.ipiv[j] = ipiv[(k+j)*inci] - k - 1;
-        }
 
         unsigned long long k_offset = offset + k*ldda;
 
-        laswpOp(EnqueueArgs(getQueue(), global, local),
-                n, inObj, k_offset, ldda, params);
+        laswpOp(EnqueueArgs(getQueue(), global, local), n, inObj, k_offset, ldda, params);
     }
-
 }
-
 }
 }
diff --git a/src/backend/opencl/kernel/lookup.hpp b/src/backend/opencl/kernel/lookup.hpp
index 995dffbadf..9528680366 100644
--- a/src/backend/opencl/kernel/lookup.hpp
+++ b/src/backend/opencl/kernel/lookup.hpp
@@ -12,8 +12,7 @@
 #include <program.hpp>
 #include <traits.hpp>
 #include <string>
-#include <mutex>
-#include <map>
+#include <cache.hpp>
 #include <dispatch.hpp>
 #include <Param.hpp>
 #include <debug_opencl.hpp>
@@ -28,59 +27,57 @@ using std::string;
 
 namespace opencl
 {
-
 namespace kernel
 {
-
 static const int THREADS_X = 32;
 static const int THREADS_Y = 8;
 
 template<typename in_t, typename idx_t, unsigned dim>
 void lookup(Param out, const Param in, const Param indices, int nDims)
 {
-    static std::once_flag compileFlags[DeviceManager::MAX_DEVICES];
-    static std::map<int, Program*>  aiProgs;
-    static std::map<int, Kernel*> aiKernels;
+    std::string refName = std::string("lookupND_") +
+        std::string(dtype_traits<in_t>::getName()) +
+        std::string(dtype_traits<idx_t>::getName()) +
+        std::to_string(dim);
 
     int device = getActiveDeviceId();
-
-    std::call_once( compileFlags[device], [device] () {
-                std::ostringstream options;
-                options << " -D in_t=" << dtype_traits<in_t>::getName()
-                        << " -D idx_t=" << dtype_traits<idx_t>::getName()
-                        << " -D DIM=" <<dim;
-
-                if (std::is_same<in_t, double>::value ||
-                    std::is_same<in_t, cdouble>::value ||
-                    std::is_same<idx_t, double>::value) {
-                    options << " -D USE_DOUBLE";
-                }
-
-                Program prog;
-                buildProgram(prog, lookup_cl, lookup_cl_len, options.str());
-                aiProgs[device]   = new Program(prog);
-                aiKernels[device] = new Kernel(*aiProgs[device], "lookupND");
-            });
+    kc_entry_t entry = kernelCache(device, refName);
+
+    if (entry.prog==0 && entry.ker==0) {
+        std::ostringstream options;
+        options << " -D in_t=" << dtype_traits<in_t>::getName()
+                << " -D idx_t=" << dtype_traits<idx_t>::getName()
+                << " -D DIM=" <<dim;
+
+        if (std::is_same<in_t, double>::value ||
+                std::is_same<in_t, cdouble>::value ||
+                std::is_same<idx_t, double>::value) {
+            options << " -D USE_DOUBLE";
+        }
+
+        const char* ker_strs[] = {lookup_cl};
+        const int   ker_lens[] = {lookup_cl_len};
+        Program prog;
+        buildProgram(prog, 1, ker_strs, ker_lens, options.str());
+        entry.prog = new Program(prog);
+        entry.ker  = new Kernel(*entry.prog, "lookupND");
+
+        addKernelToCache(device, refName, entry);
+    }
 
     NDRange local(THREADS_X, THREADS_Y);
 
     int blk_x = divup(out.info.dims[0], THREADS_X);
     int blk_y = divup(out.info.dims[1], THREADS_Y);
 
-    NDRange global(blk_x * out.info.dims[2] * THREADS_X,
-                    blk_y * out.info.dims[3] * THREADS_Y);
+    NDRange global(blk_x * out.info.dims[2] * THREADS_X, blk_y * out.info.dims[3] * THREADS_Y);
 
-    auto arrIdxOp = KernelFunctor<Buffer, KParam,
-                                Buffer, KParam,
-                                Buffer, KParam,
-                                int, int>(*aiKernels[device]);
+    auto arrIdxOp = KernelFunctor<Buffer, KParam, Buffer, KParam, Buffer, KParam, int, int>(*entry.ker);
 
     arrIdxOp(EnqueueArgs(getQueue(), global, local),
-            *out.data, out.info, *in.data, in.info, *indices.data, indices.info, blk_x, blk_y);
+             *out.data, out.info, *in.data, in.info, *indices.data, indices.info, blk_x, blk_y);
 
     CL_DEBUG_FINISH(getQueue());
 }
-
 }
-
 }
diff --git a/src/backend/opencl/kernel/lu_split.hpp b/src/backend/opencl/kernel/lu_split.hpp
index 6784039614..858b931660 100644
--- a/src/backend/opencl/kernel/lu_split.hpp
+++ b/src/backend/opencl/kernel/lu_split.hpp
@@ -12,8 +12,7 @@
 #include <program.hpp>
 #include <traits.hpp>
 #include <string>
-#include <mutex>
-#include <map>
+#include <cache.hpp>
 #include <dispatch.hpp>
 #include <Param.hpp>
 #include <debug_opencl.hpp>
@@ -31,10 +30,8 @@ using af::scalar_to_option;
 
 namespace opencl
 {
-
 namespace kernel
 {
-
 // Kernel Launch Config Values
 static const unsigned TX = 32;
 static const unsigned TY = 8;
@@ -44,51 +41,46 @@ static const unsigned TILEY = 32;
 template<typename T, bool same_dims>
 void lu_split_launcher(Param lower, Param upper, const Param in)
 {
-    static std::once_flag compileFlags[DeviceManager::MAX_DEVICES];
-    static std::map<int, Program*>  splitProgs;
-    static std::map<int, Kernel*> splitKernels;
+    std::string refName = std::string("lu_split_kernel_") +
+        std::string(dtype_traits<T>::getName()) +
+        std::to_string(same_dims);
 
     int device = getActiveDeviceId();
-
-    std::call_once(compileFlags[device], [device] () {
-
-            std::ostringstream options;
-            options << " -D T=" << dtype_traits<T>::getName()
-                    << " -D same_dims=" << same_dims
-                    << " -D ZERO=(T)(" << scalar_to_option(scalar<T>(0)) << ")"
-                    << " -D ONE=(T)(" << scalar_to_option(scalar<T>(1)) << ")";
-
-            if (std::is_same<T, double>::value ||
-                std::is_same<T, cdouble>::value) {
-                options << " -D USE_DOUBLE";
-            }
-
-            cl::Program prog;
-            buildProgram(prog, lu_split_cl, lu_split_cl_len, options.str());
-            splitProgs[device] = new Program(prog);
-
-            splitKernels[device] = new Kernel(*splitProgs[device], "lu_split_kernel");
-        });
-
+    kc_entry_t entry = kernelCache(device, refName);
+
+    if (entry.prog==0 && entry.ker==0) {
+        std::ostringstream options;
+        options << " -D T=" << dtype_traits<T>::getName()
+            << " -D same_dims=" << same_dims
+            << " -D ZERO=(T)(" << scalar_to_option(scalar<T>(0)) << ")"
+            << " -D ONE=(T)(" << scalar_to_option(scalar<T>(1)) << ")";
+
+        if (std::is_same<T, double>::value || std::is_same<T, cdouble>::value)
+            options << " -D USE_DOUBLE";
+
+        const char* ker_strs[] = {lu_split_cl};
+        const int   ker_lens[] = {lu_split_cl_len};
+        Program prog;
+        buildProgram(prog, 1, ker_strs, ker_lens, options.str());
+        entry.prog = new Program(prog);
+        entry.ker  = new Kernel(*entry.prog, "lu_split_kernel");
+
+        addKernelToCache(device, refName, entry);
+    }
 
     NDRange local(TX, TY);
 
     int groups_x = divup(in.info.dims[0], TILEX);
     int groups_y = divup(in.info.dims[1], TILEY);
 
-    NDRange global(groups_x * local[0] * in.info.dims[2],
-                    groups_y * local[1] * in.info.dims[3]);
+    NDRange global(groups_x * local[0] * in.info.dims[2], groups_y * local[1] * in.info.dims[3]);
 
-    auto lu_split_op = KernelFunctor<Buffer, const KParam,
-                                    Buffer, const KParam,
-                                    const Buffer, const KParam,
-                                    const int, const int> (*splitKernels[device]);
+    auto lu_split_op = KernelFunctor<Buffer, const KParam, Buffer, const KParam,
+                                    const Buffer, const KParam, const int, const int> (*entry.ker);
 
     lu_split_op(EnqueueArgs(getQueue(), global, local),
-                *lower.data, lower.info,
-                *upper.data, upper.info,
-                *in.data, in.info,
-                groups_x, groups_y);
+                *lower.data, lower.info, *upper.data, upper.info,
+                *in.data, in.info, groups_x, groups_y);
 
     CL_DEBUG_FINISH(getQueue());
 }
@@ -106,7 +98,5 @@ void lu_split(Param lower, Param upper, const Param in)
         lu_split_launcher<T, false>(lower, upper, in);
     }
 }
-
 }
-
 }
diff --git a/src/backend/opencl/kernel/match_template.hpp b/src/backend/opencl/kernel/match_template.hpp
index 4922abb784..4cd5965371 100644
--- a/src/backend/opencl/kernel/match_template.hpp
+++ b/src/backend/opencl/kernel/match_template.hpp
@@ -12,8 +12,7 @@
 #include <program.hpp>
 #include <traits.hpp>
 #include <string>
-#include <mutex>
-#include <map>
+#include <cache.hpp>
 #include <dispatch.hpp>
 #include <Param.hpp>
 #include <debug_opencl.hpp>
@@ -28,46 +27,49 @@ using std::string;
 
 namespace opencl
 {
-
 namespace kernel
 {
-
 static const int THREADS_X = 16;
 static const int THREADS_Y = 16;
 
 template<typename inType, typename outType, af_match_type mType, bool needMean>
 void matchTemplate(Param out, const Param srch, const Param tmplt)
 {
-    static std::once_flag compileFlags[DeviceManager::MAX_DEVICES];
-    static std::map<int, Program*>  mtProgs;
-    static std::map<int, Kernel*> mtKernels;
+    std::string refName = std::string("matchTemplate_") +
+        std::string(dtype_traits<inType>::getName()) +
+        std::string(dtype_traits<outType>::getName()) +
+        std::to_string(mType) + std::to_string(needMean);
 
     int device = getActiveDeviceId();
-
-    std::call_once( compileFlags[device], [device] () {
-
-            std::ostringstream options;
-            options << " -D inType="  << dtype_traits<inType>::getName()
-                    << " -D outType=" << dtype_traits<outType>::getName()
-                    << " -D MATCH_T=" << mType
-                    << " -D NEEDMEAN="<< needMean
-                    << " -D AF_SAD="  << AF_SAD
-                    << " -D AF_ZSAD=" << AF_ZSAD
-                    << " -D AF_LSAD=" << AF_LSAD
-                    << " -D AF_SSD="  << AF_SSD
-                    << " -D AF_ZSSD=" << AF_ZSSD
-                    << " -D AF_LSSD=" << AF_LSSD
-                    << " -D AF_NCC="  << AF_NCC
-                    << " -D AF_ZNCC=" << AF_ZNCC
-                    << " -D AF_SHD="  << AF_SHD;
-            if (std::is_same<outType, double>::value) {
-                options << " -D USE_DOUBLE";
-            }
-            Program prog;
-            buildProgram(prog, matchTemplate_cl, matchTemplate_cl_len, options.str());
-            mtProgs[device]   = new Program(prog);
-            mtKernels[device] = new Kernel(*mtProgs[device], "matchTemplate");
-        });
+    kc_entry_t entry = kernelCache(device, refName);
+
+    if (entry.prog==0 && entry.ker==0) {
+        std::ostringstream options;
+        options << " -D inType="  << dtype_traits<inType>::getName()
+                << " -D outType=" << dtype_traits<outType>::getName()
+                << " -D MATCH_T=" << mType
+                << " -D NEEDMEAN="<< needMean
+                << " -D AF_SAD="  << AF_SAD
+                << " -D AF_ZSAD=" << AF_ZSAD
+                << " -D AF_LSAD=" << AF_LSAD
+                << " -D AF_SSD="  << AF_SSD
+                << " -D AF_ZSSD=" << AF_ZSSD
+                << " -D AF_LSSD=" << AF_LSSD
+                << " -D AF_NCC="  << AF_NCC
+                << " -D AF_ZNCC=" << AF_ZNCC
+                << " -D AF_SHD="  << AF_SHD;
+        if (std::is_same<outType, double>::value)
+            options << " -D USE_DOUBLE";
+
+        const char* ker_strs[] = {matchTemplate_cl};
+        const int   ker_lens[] = {matchTemplate_cl_len};
+        Program prog;
+        buildProgram(prog, 1, ker_strs, ker_lens, options.str());
+        entry.prog = new Program(prog);
+        entry.ker  = new Kernel(*entry.prog, "matchTemplate");
+
+        addKernelToCache(device, refName, entry);
+    }
 
     NDRange local(THREADS_X, THREADS_Y);
 
@@ -76,17 +78,13 @@ void matchTemplate(Param out, const Param srch, const Param tmplt)
 
     NDRange global(blk_x * srch.info.dims[2] * THREADS_X, blk_y * srch.info.dims[3] * THREADS_Y);
 
-    auto matchImgOp = KernelFunctor<Buffer, KParam,
-                                    Buffer, KParam,
-                                    Buffer, KParam,
-                                    int, int> (*mtKernels[device]);
+    auto matchImgOp = KernelFunctor<Buffer, KParam, Buffer, KParam, Buffer, KParam,
+                                    int, int> (*entry.ker);
 
     matchImgOp(EnqueueArgs(getQueue(), global, local),
-                *out.data, out.info, *srch.data, srch.info, *tmplt.data, tmplt.info, blk_x, blk_y);
+               *out.data, out.info, *srch.data, srch.info, *tmplt.data, tmplt.info, blk_x, blk_y);
 
     CL_DEBUG_FINISH(getQueue());
 }
-
 }
-
 }
diff --git a/src/backend/opencl/kernel/meanshift.hpp b/src/backend/opencl/kernel/meanshift.hpp
index ddab5ed330..cb82021c3d 100644
--- a/src/backend/opencl/kernel/meanshift.hpp
+++ b/src/backend/opencl/kernel/meanshift.hpp
@@ -12,8 +12,7 @@
 #include <program.hpp>
 #include <traits.hpp>
 #include <string>
-#include <mutex>
-#include <map>
+#include <cache.hpp>
 #include <algorithm>
 #include <dispatch.hpp>
 #include <Param.hpp>
@@ -30,43 +29,39 @@ using std::string;
 
 namespace opencl
 {
-
 namespace kernel
 {
-
 static const int THREADS_X = 16;
 static const int THREADS_Y = 16;
 
 template<typename T, bool is_color>
 void meanshift(Param out, const Param in, float s_sigma, float c_sigma, uint iter)
 {
-    static std::once_flag compileFlags[DeviceManager::MAX_DEVICES];
-    static std::map<int, Program*> msProgs;
-    static std::map<int, Kernel*> msKernels;
+    std::string refName = std::string("meanshift_") +
+        std::string(dtype_traits<T>::getName()) + std::to_string(is_color);
 
     int device = getActiveDeviceId();
+    kc_entry_t entry = kernelCache(device, refName);
+
+    if (entry.prog==0 && entry.ker==0) {
+        std::ostringstream options;
+        options << " -D T=" << dtype_traits<T>::getName()
+                << " -D MAX_CHANNELS=" << (is_color ? 3 : 1);
+        if (std::is_same<T, double>::value || std::is_same<T, cdouble>::value)
+            options << " -D USE_DOUBLE";
+
+        const char* ker_strs[] = {meanshift_cl};
+        const int   ker_lens[] = {meanshift_cl_len};
+        Program prog;
+        buildProgram(prog, 1, ker_strs, ker_lens, options.str());
+        entry.prog = new Program(prog);
+        entry.ker  = new Kernel(*entry.prog, "meanshift");
 
-    std::call_once( compileFlags[device], [device] () {
-                std::ostringstream options;
-                options << " -D T=" << dtype_traits<T>::getName()
-                        << " -D MAX_CHANNELS=" << (is_color ? 3 : 1);
-                if (std::is_same<T, double>::value ||
-                    std::is_same<T, cdouble>::value) {
-                    options << " -D USE_DOUBLE";
-                }
-                Program prog;
-                buildProgram(prog, meanshift_cl, meanshift_cl_len, options.str());
-                msProgs[device]   = new Program(prog);
-                msKernels[device] = new Kernel(*msProgs[device], "meanshift");
-            });
-
-    auto meanshiftOp = KernelFunctor<Buffer, KParam,
-                                    Buffer, KParam,
-                                    LocalSpaceArg,
-                                    int, float,
-                                    int, float,
-                                    unsigned, int, int
-                                  >(*msKernels[device]);
+        addKernelToCache(device, refName, entry);
+    }
+
+    auto meanshiftOp = KernelFunctor<Buffer, KParam, Buffer, KParam, LocalSpaceArg,
+                                    int, float, int, float, unsigned, int, int >(*entry.ker);
 
     NDRange local(THREADS_X, THREADS_Y);
 
@@ -92,7 +87,5 @@ void meanshift(Param out, const Param in, float s_sigma, float c_sigma, uint ite
 
     CL_DEBUG_FINISH(getQueue());
 }
-
 }
-
 }
diff --git a/src/backend/opencl/kernel/medfilt.hpp b/src/backend/opencl/kernel/medfilt.hpp
index 493071d7e9..3a7a471825 100644
--- a/src/backend/opencl/kernel/medfilt.hpp
+++ b/src/backend/opencl/kernel/medfilt.hpp
@@ -13,8 +13,7 @@
 #include <program.hpp>
 #include <traits.hpp>
 #include <string>
-#include <mutex>
-#include <map>
+#include <cache.hpp>
 #include <dispatch.hpp>
 #include <Param.hpp>
 #include <debug_opencl.hpp>
@@ -29,10 +28,8 @@ using std::string;
 
 namespace opencl
 {
-
 namespace kernel
 {
-
 static const int MAX_MEDFILTER2_LEN = 15;
 static const int MAX_MEDFILTER1_LEN = 121;
 
@@ -42,50 +39,47 @@ static const int THREADS_Y = 16;
 template<typename T, af_border_type pad>
 void medfilt1(Param out, const Param in, unsigned w_wid)
 {
-    static std::once_flag compileFlags[DeviceManager::MAX_DEVICES];
-    static std::map<int, Program*>  mfProgs;
-    static std::map<int, Kernel*> mfKernels;
+    std::string refName = std::string("medfilt1_") +
+        std::string(dtype_traits<T>::getName()) + std::to_string(pad);
 
     int device = getActiveDeviceId();
-
-    std::call_once( compileFlags[device], [device, w_wid] () {
-
-            const int ARR_SIZE = (w_wid-w_wid/2) + 1;
-
-            std::ostringstream options;
-            options << " -D T=" << dtype_traits<T>::getName()
-                    << " -D pad="<< pad
-                    << " -D AF_PAD_ZERO="<< AF_PAD_ZERO
-                    << " -D AF_PAD_SYM="<< AF_PAD_SYM
-                    << " -D ARR_SIZE="<< ARR_SIZE
-                    << " -D w_wid=" << w_wid;
-            if (std::is_same<T, double>::value ||
-                std::is_same<T, cdouble>::value) {
-                options << " -D USE_DOUBLE";
-            }
-            Program prog;
-            buildProgram(prog, medfilt1_cl, medfilt1_cl_len, options.str());
-            mfProgs[device]   = new Program(prog);
-            mfKernels[device] = new Kernel(*mfProgs[device], "medfilt1");
-        });
+    kc_entry_t entry = kernelCache(device, refName);
+
+    if (entry.prog==0 && entry.ker==0) {
+        const int ARR_SIZE = (w_wid-w_wid/2) + 1;
+
+        std::ostringstream options;
+        options << " -D T=" << dtype_traits<T>::getName()
+                << " -D pad="<< pad
+                << " -D AF_PAD_ZERO="<< AF_PAD_ZERO
+                << " -D AF_PAD_SYM="<< AF_PAD_SYM
+                << " -D ARR_SIZE="<< ARR_SIZE
+                << " -D w_wid=" << w_wid;
+        if (std::is_same<T, double>::value || std::is_same<T, cdouble>::value)
+            options << " -D USE_DOUBLE";
+
+        const char* ker_strs[] = {medfilt1_cl};
+        const int   ker_lens[] = {medfilt1_cl_len};
+        Program prog;
+        buildProgram(prog, 1, ker_strs, ker_lens, options.str());
+        entry.prog = new Program(prog);
+        entry.ker  = new Kernel(*entry.prog, "medfilt1");
+
+        addKernelToCache(device, refName, entry);
+    }
 
     NDRange local(THREADS_X, 1, 1);
 
     int blk_x = divup(in.info.dims[0], THREADS_X);
 
-    NDRange global(blk_x * in.info.dims[1] * THREADS_X,
-                                        in.info.dims[2],
-                                        in.info.dims[3]);
+    NDRange global(blk_x * in.info.dims[1] * THREADS_X, in.info.dims[2], in.info.dims[3]);
 
-    auto medfiltOp = KernelFunctor<Buffer, KParam,
-                                    Buffer, KParam,
-                                    cl::LocalSpaceArg,
-                                    int> (*mfKernels[device]);
+    auto medfiltOp = KernelFunctor<Buffer, KParam, Buffer, KParam, cl::LocalSpaceArg, int> (*entry.ker);
 
     size_t loc_size = (THREADS_X+w_wid-1)*sizeof(T);
 
     medfiltOp(EnqueueArgs(getQueue(), global, local),
-                *out.data, out.info, *in.data, in.info, cl::Local(loc_size), blk_x);
+              *out.data, out.info, *in.data, in.info, cl::Local(loc_size), blk_x);
 
     CL_DEBUG_FINISH(getQueue());
 }
@@ -93,55 +87,53 @@ void medfilt1(Param out, const Param in, unsigned w_wid)
 template<typename T, af_border_type pad, unsigned w_len, unsigned w_wid>
 void medfilt2(Param out, const Param in)
 {
-    static std::once_flag compileFlags[DeviceManager::MAX_DEVICES];
-    static std::map<int, Program*>  mfProgs;
-    static std::map<int, Kernel*> mfKernels;
+    std::string refName = std::string("medfilt2_") +
+        std::string(dtype_traits<T>::getName()) +
+        std::to_string(pad) + std::to_string(w_len) + std::to_string(w_wid);
 
     int device = getActiveDeviceId();
-
-    std::call_once( compileFlags[device], [device] () {
-
-            const int ARR_SIZE = w_len * (w_wid-w_wid/2);
-
-            std::ostringstream options;
-            options << " -D T=" << dtype_traits<T>::getName()
-                    << " -D pad="<< pad
-                    << " -D AF_PAD_ZERO="<< AF_PAD_ZERO
-                    << " -D AF_PAD_SYM="<< AF_PAD_SYM
-                    << " -D ARR_SIZE="<< ARR_SIZE
-                    << " -D w_len="<< w_len
-                    << " -D w_wid=" << w_wid;
-            if (std::is_same<T, double>::value ||
-                std::is_same<T, cdouble>::value) {
-                options << " -D USE_DOUBLE";
-            }
-            Program prog;
-            buildProgram(prog, medfilt2_cl, medfilt2_cl_len, options.str());
-            mfProgs[device]   = new Program(prog);
-            mfKernels[device] = new Kernel(*mfProgs[device], "medfilt2");
-        });
+    kc_entry_t entry = kernelCache(device, refName);
+
+    if (entry.prog==0 && entry.ker==0) {
+        const int ARR_SIZE = w_len * (w_wid-w_wid/2);
+
+        std::ostringstream options;
+        options << " -D T=" << dtype_traits<T>::getName()
+                << " -D pad="<< pad
+                << " -D AF_PAD_ZERO="<< AF_PAD_ZERO
+                << " -D AF_PAD_SYM="<< AF_PAD_SYM
+                << " -D ARR_SIZE="<< ARR_SIZE
+                << " -D w_len="<< w_len
+                << " -D w_wid=" << w_wid;
+        if (std::is_same<T, double>::value || std::is_same<T, cdouble>::value)
+            options << " -D USE_DOUBLE";
+
+        const char* ker_strs[] = {medfilt2_cl};
+        const int   ker_lens[] = {medfilt2_cl_len};
+        Program prog;
+        buildProgram(prog, 1, ker_strs, ker_lens, options.str());
+        entry.prog = new Program(prog);
+        entry.ker  = new Kernel(*entry.prog, "medfilt2");
+
+        addKernelToCache(device, refName, entry);
+    }
 
     NDRange local(THREADS_X, THREADS_Y);
 
     int blk_x = divup(in.info.dims[0], THREADS_X);
     int blk_y = divup(in.info.dims[1], THREADS_Y);
 
-    NDRange global(blk_x * in.info.dims[2] * THREADS_X,
-                    blk_y * in.info.dims[3] * THREADS_Y);
+    NDRange global(blk_x * in.info.dims[2] * THREADS_X, blk_y * in.info.dims[3] * THREADS_Y);
 
-    auto medfiltOp = KernelFunctor<Buffer, KParam,
-                                  Buffer, KParam,
-                                  cl::LocalSpaceArg,
-                                  int, int> (*mfKernels[device]);
+    auto medfiltOp = KernelFunctor<Buffer, KParam, Buffer, KParam, cl::LocalSpaceArg,
+                                   int, int> (*entry.ker);
 
     size_t loc_size = (THREADS_X+w_len-1)*(THREADS_Y+w_wid-1)*sizeof(T);
 
     medfiltOp(EnqueueArgs(getQueue(), global, local),
-                *out.data, out.info, *in.data, in.info, cl::Local(loc_size), blk_x, blk_y);
+              *out.data, out.info, *in.data, in.info, cl::Local(loc_size), blk_x, blk_y);
 
     CL_DEBUG_FINISH(getQueue());
 }
-
 }
-
 }
diff --git a/src/backend/opencl/kernel/memcopy.hpp b/src/backend/opencl/kernel/memcopy.hpp
index 7b413b0a10..6ee1733ecb 100644
--- a/src/backend/opencl/kernel/memcopy.hpp
+++ b/src/backend/opencl/kernel/memcopy.hpp
@@ -14,8 +14,8 @@
 #include <traits.hpp>
 #include <sstream>
 #include <string>
-#include <map>
 #include <algorithm>
+#include <cache.hpp>
 #include <dispatch.hpp>
 #include <Param.hpp>
 #include <debug_opencl.hpp>
@@ -29,135 +29,135 @@ using std::string;
 
 namespace opencl
 {
-
 namespace kernel
 {
+typedef struct
+{
+    dim_t dim[4];
+} dims_t;
+
+static const uint DIM0 = 32;
+static const uint DIM1 =  8;
+
+template<typename T>
+void memcopy(cl::Buffer out, const dim_t *ostrides,
+             const cl::Buffer in, const dim_t *idims,
+             const dim_t *istrides, int offset, uint ndims)
+{
+    std::string refName = std::string("memcopy_") + std::string(dtype_traits<T>::getName());
+
+    int device = getActiveDeviceId();
+    kc_entry_t entry = kernelCache(device, refName);
+
+    if (entry.prog==0 && entry.ker==0) {
+        std::ostringstream options;
 
-    typedef struct
-    {
-        dim_t dim[4];
-    } dims_t;
-
-    static const uint DIM0 = 32;
-    static const uint DIM1 =  8;
-
-    template<typename T>
-    void memcopy(cl::Buffer out, const dim_t *ostrides,
-                 const cl::Buffer in, const dim_t *idims,
-                 const dim_t *istrides, int offset, uint ndims)
-    {
-        static std::once_flag compileFlags[DeviceManager::MAX_DEVICES];
-        static std::map<int, Program*>    cpyProgs;
-        static std::map<int, Kernel*>   cpyKernels;
-
-        int device = getActiveDeviceId();
-
-        std::call_once(compileFlags[device], [&]() {
-            std::ostringstream options;
-            options << " -D T=" << dtype_traits<T>::getName();
-            if (std::is_same<T, double>::value ||
-                std::is_same<T, cdouble>::value) {
-                options << " -D USE_DOUBLE";
-            }
-            Program prog;
-            buildProgram(prog, memcopy_cl, memcopy_cl_len, options.str());
-            cpyProgs[device]   = new Program(prog);
-            cpyKernels[device] = new Kernel(*cpyProgs[device], "memcopy_kernel");
-        });
-
-        dims_t _ostrides = {{ostrides[0], ostrides[1], ostrides[2], ostrides[3]}};
-        dims_t _istrides = {{istrides[0], istrides[1], istrides[2], istrides[3]}};
-        dims_t _idims = {{idims[0], idims[1], idims[2], idims[3]}};
-
-        size_t local_size[2] = {DIM0, DIM1};
-        if (ndims == 1) {
-            local_size[0] *= local_size[1];
-            local_size[1]  = 1;
-        }
-
-        int groups_0 = divup(idims[0], local_size[0]);
-        int groups_1 = divup(idims[1], local_size[1]);
-
-        NDRange local(local_size[0], local_size[1]);
-        NDRange global(groups_0 * idims[2] * local_size[0],
-                        groups_1 * idims[3] * local_size[1]);
-
-        auto memcopy_kernel = KernelFunctor< Buffer, dims_t,
-                                            Buffer, dims_t,
-                                            dims_t, int,
-                                            int, int >(*cpyKernels[device]);
-
-        memcopy_kernel(EnqueueArgs(getQueue(), global, local),
-            out, _ostrides, in, _idims, _istrides, offset, groups_0, groups_1);
-        CL_DEBUG_FINISH(getQueue());
+        options << " -D T=" << dtype_traits<T>::getName();
+        if (std::is_same<T, double>::value || std::is_same<T, cdouble>::value)
+            options << " -D USE_DOUBLE";
+
+        const char* ker_strs[] = {memcopy_cl};
+        const int   ker_lens[] = {memcopy_cl_len};
+        Program prog;
+        buildProgram(prog, 1, ker_strs, ker_lens, options.str());
+        entry.prog = new Program(prog);
+        entry.ker  = new Kernel(*entry.prog, "memcopy_kernel");
+
+        addKernelToCache(device, refName, entry);
     }
 
-    template<typename inType, typename outType, bool same_dims>
-    void copy(Param dst, const Param src, int ndims, outType default_value, double factor)
-    {
-        static std::once_flag compileFlags[DeviceManager::MAX_DEVICES];
-        static std::map<int, Program*>    cpyProgs;
-        static std::map<int, Kernel*>   cpyKernels;
-
-        int device = getActiveDeviceId();
-
-        std::call_once(compileFlags[device], [&]() {
-
-                    std::ostringstream options;
-                    options << " -D inType=" << dtype_traits<inType>::getName()
-                        << " -D outType=" << dtype_traits<outType>::getName()
-                        << " -D inType_" << dtype_traits<inType>::getName()
-                        << " -D outType_" << dtype_traits<outType>::getName()
-                        << " -D SAME_DIMS=" << same_dims;
-                    if (std::is_same<inType, double>::value  ||
-                        std::is_same<inType, cdouble>::value ||
-                        std::is_same<outType, double>::value ||
-                        std::is_same<outType, cdouble>::value) {
-                        options << " -D USE_DOUBLE";
-                    }
-
-                    Program prog;
-                    buildProgram(prog, copy_cl, copy_cl_len, options.str());
-                    cpyProgs[device]   = new Program(prog);
-                    cpyKernels[device] = new Kernel(*cpyProgs[device], "copy");
-                });
-
-        NDRange local(DIM0, DIM1);
-        size_t local_size[] = {DIM0, DIM1};
+    dims_t _ostrides = {{ostrides[0], ostrides[1], ostrides[2], ostrides[3]}};
+    dims_t _istrides = {{istrides[0], istrides[1], istrides[2], istrides[3]}};
+    dims_t _idims = {{idims[0], idims[1], idims[2], idims[3]}};
 
+    size_t local_size[2] = {DIM0, DIM1};
+    if (ndims == 1) {
         local_size[0] *= local_size[1];
-        if (ndims == 1) {
-            local_size[1] = 1;
-        }
-
-        int blk_x = divup(dst.info.dims[0], local_size[0]);
-        int blk_y = divup(dst.info.dims[1], local_size[1]);
-
-        NDRange global(blk_x * dst.info.dims[2] * DIM0,
-                blk_y * dst.info.dims[3] * DIM1);
-
-        dims_t trgt_dims;
-        if (same_dims) {
-            trgt_dims= {{dst.info.dims[0], dst.info.dims[1], dst.info.dims[2], dst.info.dims[3]}};
-        } else {
-            dim_t trgt_l = std::min(dst.info.dims[3], src.info.dims[3]);
-            dim_t trgt_k = std::min(dst.info.dims[2], src.info.dims[2]);
-            dim_t trgt_j = std::min(dst.info.dims[1], src.info.dims[1]);
-            dim_t trgt_i = std::min(dst.info.dims[0], src.info.dims[0]);
-            trgt_dims= {{trgt_i, trgt_j, trgt_k, trgt_l}};
-        }
-
-        auto copyOp = KernelFunctor<Buffer, KParam, Buffer, KParam,
-                                  outType, float, dims_t,
-                                  int, int
-                                  >(*cpyKernels[device]);
-
-        copyOp(EnqueueArgs(getQueue(), global, local),
-                *dst.data, dst.info, *src.data, src.info,
-                default_value, (float)factor, trgt_dims, blk_x, blk_y);
-        CL_DEBUG_FINISH(getQueue());
+        local_size[1]  = 1;
     }
 
+    int groups_0 = divup(idims[0], local_size[0]);
+    int groups_1 = divup(idims[1], local_size[1]);
+
+    NDRange local(local_size[0], local_size[1]);
+    NDRange global(groups_0 * idims[2] * local_size[0], groups_1 * idims[3] * local_size[1]);
+
+    auto memCpyOp = KernelFunctor< Buffer, dims_t, Buffer, dims_t,
+                                   dims_t, int, int, int >(*entry.ker);
+
+    memCpyOp(EnqueueArgs(getQueue(), global, local),
+             out, _ostrides, in, _idims, _istrides, offset, groups_0, groups_1);
+
+    CL_DEBUG_FINISH(getQueue());
 }
 
+template<typename inType, typename outType, bool same_dims>
+void copy(Param dst, const Param src, int ndims, outType default_value, double factor)
+{
+    std::string refName =
+        std::string("copy_") +
+        std::string(dtype_traits<inType>::getName()) +
+        std::string(dtype_traits<outType>::getName()) +
+        std::to_string(same_dims);
+
+    int device = getActiveDeviceId();
+    kc_entry_t entry = kernelCache(device, refName);
+
+    if (entry.prog==0 && entry.ker==0) {
+        std::ostringstream options;
+
+        options << " -D inType="    << dtype_traits<inType>::getName()
+                << " -D outType="   << dtype_traits<outType>::getName()
+                << " -D inType_"    << dtype_traits<inType>::getName()
+                << " -D outType_"   << dtype_traits<outType>::getName()
+                << " -D SAME_DIMS=" << same_dims;
+
+        if (std::is_same<inType, double>::value  || std::is_same<inType, cdouble>::value ||
+            std::is_same<outType, double>::value || std::is_same<outType, cdouble>::value)
+            options << " -D USE_DOUBLE";
+
+        const char* ker_strs[] = {copy_cl};
+        const int   ker_lens[] = {copy_cl_len};
+        Program prog;
+        buildProgram(prog, 1, ker_strs, ker_lens, options.str());
+        entry.prog = new Program(prog);
+        entry.ker  = new Kernel(*entry.prog, "copy");
+
+        addKernelToCache(device, refName, entry);
+    }
+
+    NDRange local(DIM0, DIM1);
+    size_t local_size[] = {DIM0, DIM1};
+
+    local_size[0] *= local_size[1];
+    if (ndims == 1) {
+        local_size[1] = 1;
+    }
+
+    int blk_x = divup(dst.info.dims[0], local_size[0]);
+    int blk_y = divup(dst.info.dims[1], local_size[1]);
+
+    NDRange global(blk_x * dst.info.dims[2] * DIM0, blk_y * dst.info.dims[3] * DIM1);
+
+    dims_t trgt_dims;
+    if (same_dims) {
+        trgt_dims= {{dst.info.dims[0], dst.info.dims[1], dst.info.dims[2], dst.info.dims[3]}};
+    } else {
+        dim_t trgt_l = std::min(dst.info.dims[3], src.info.dims[3]);
+        dim_t trgt_k = std::min(dst.info.dims[2], src.info.dims[2]);
+        dim_t trgt_j = std::min(dst.info.dims[1], src.info.dims[1]);
+        dim_t trgt_i = std::min(dst.info.dims[0], src.info.dims[0]);
+        trgt_dims= {{trgt_i, trgt_j, trgt_k, trgt_l}};
+    }
+
+    auto copyOp = KernelFunctor< Buffer, KParam, Buffer, KParam,
+                                 outType, float, dims_t, int, int >(*entry.ker);
+
+    copyOp(EnqueueArgs(getQueue(), global, local),
+           *dst.data, dst.info, *src.data, src.info,
+           default_value, (float)factor, trgt_dims, blk_x, blk_y);
+
+    CL_DEBUG_FINISH(getQueue());
+}
+}
 }
diff --git a/src/backend/opencl/kernel/morph.hpp b/src/backend/opencl/kernel/morph.hpp
index 3f129ffa9e..8c84033b56 100644
--- a/src/backend/opencl/kernel/morph.hpp
+++ b/src/backend/opencl/kernel/morph.hpp
@@ -12,8 +12,7 @@
 #include <program.hpp>
 #include <traits.hpp>
 #include <string>
-#include <mutex>
-#include <map>
+#include <cache.hpp>
 #include <dispatch.hpp>
 #include <Param.hpp>
 #include <debug_opencl.hpp>
@@ -32,10 +31,8 @@ using std::string;
 
 namespace opencl
 {
-
 namespace kernel
 {
-
 static const int THREADS_X = 16;
 static const int THREADS_Y = 16;
 
@@ -44,47 +41,50 @@ static const int CUBE_Y    =  8;
 static const int CUBE_Z    =  4;
 
 template<typename T, bool isDilation, int windLen>
-void morph(Param         out,
-        const Param      in,
-        const Param      mask)
+std::string generateOptionsString()
 {
-    static std::once_flag compileFlags[DeviceManager::MAX_DEVICES];
-    static std::map<int, Program*> morProgs;
-    static std::map<int, Kernel*> morKernels;
+    ToNumStr<T> toNumStr;
+    T init = isDilation ? Binary<T, af_max_t>().init() : Binary<T, af_min_t>().init();
+    std::ostringstream options;
+    options << " -D T=" << dtype_traits<T>::getName()
+        << " -D isDilation="<< isDilation
+        << " -D init=" << toNumStr(init)
+        << " -D windLen=" << windLen;
+    if (std::is_same<T, double>::value || std::is_same<T, cdouble>::value)
+        options << " -D USE_DOUBLE";
+    return options.str();
+}
 
-    int device = getActiveDeviceId();
+template<typename T, bool isDilation, int windLen>
+void morph(Param out, const Param in, const Param mask)
+{
+    std::string refName = std::string("morph_") +
+        std::string(dtype_traits<T>::getName()) +
+        std::to_string(isDilation) + std::to_string(windLen);
 
-    std::call_once( compileFlags[device], [device] () {
-            ToNumStr<T> toNumStr;
-            T init = isDilation ? Binary<T, af_max_t>().init() : Binary<T, af_min_t>().init();
-            std::ostringstream options;
-            options << " -D T=" << dtype_traits<T>::getName()
-                    << " -D isDilation="<< isDilation
-                    << " -D init=" << toNumStr(init)
-                    << " -D windLen=" << windLen;
-            if (std::is_same<T, double>::value ||
-                std::is_same<T, cdouble>::value) {
-                options << " -D USE_DOUBLE";
-            }
-            Program prog;
-            buildProgram(prog, morph_cl, morph_cl_len, options.str());
-            morProgs[device]   = new Program(prog);
-            morKernels[device] = new Kernel(*morProgs[device], "morph");
-        });
-
-    auto morphOp = KernelFunctor<Buffer, KParam,
-                                Buffer, KParam,
-                                Buffer, cl::LocalSpaceArg,
-                                int, int
-                              >(*morKernels[device]);
+    int device = getActiveDeviceId();
+    kc_entry_t entry = kernelCache(device, refName);
+
+    if (entry.prog==0 && entry.ker==0) {
+        std::string options = generateOptionsString<T, isDilation, windLen>();
+        const char* ker_strs[] = {morph_cl};
+        const int   ker_lens[] = {morph_cl_len};
+        Program prog;
+        buildProgram(prog, 1, ker_strs, ker_lens, options);
+        entry.prog = new Program(prog);
+        entry.ker  = new Kernel(*entry.prog, "morph");
+        addKernelToCache(device, refName, entry);
+    }
+
+    auto morphOp = KernelFunctor< Buffer, KParam, Buffer, KParam, Buffer, cl::LocalSpaceArg,
+                                  int, int >(*entry.ker);
 
     NDRange local(THREADS_X, THREADS_Y);
 
     int blk_x = divup(in.info.dims[0], THREADS_X);
     int blk_y = divup(in.info.dims[1], THREADS_Y);
     // launch batch * blk_x blocks along x dimension
-    NDRange global(blk_x * THREADS_X * in.info.dims[2],
-                    blk_y * THREADS_Y * in.info.dims[3]);
+    NDRange global(blk_x * THREADS_X * in.info.dims[2], blk_y * THREADS_Y * in.info.dims[3]);
 
     // copy mask/filter to constant memory
     cl_int se_size   = sizeof(T)*windLen*windLen;
@@ -111,34 +111,26 @@ void morph3d(Param       out,
         const Param      in,
         const Param      mask)
 {
-    static std::once_flag compileFlags[DeviceManager::MAX_DEVICES];
-    static std::map<int, Program*>  morProgs;
-    static std::map<int, Kernel*> morKernels;
+    std::string refName = std::string("morph3d_") +
+        std::string(dtype_traits<T>::getName()) +
+        std::to_string(isDilation) + std::to_string(windLen);
 
     int device = getActiveDeviceId();
-
-    std::call_once( compileFlags[device], [device] () {
-            ToNumStr<T> toNumStr;
-            T init = isDilation ? Binary<T, af_max_t>().init() : Binary<T, af_min_t>().init();
-            std::ostringstream options;
-            options << " -D T=" << dtype_traits<T>::getName()
-                    << " -D isDilation="<< isDilation
-                    << " -D init=" << toNumStr(init)
-                    << " -D windLen=" << windLen;
-            if (std::is_same<T, double>::value ||
-                std::is_same<T, cdouble>::value) {
-                options << " -D USE_DOUBLE";
-            }
-            Program prog;
-            buildProgram(prog, morph_cl, morph_cl_len, options.str());
-            morProgs[device]   = new Program(prog);
-            morKernels[device] = new Kernel(*morProgs[device], "morph3d");
-        });
-
-    auto morphOp = KernelFunctor<Buffer, KParam,
-                                Buffer, KParam,
-                                Buffer, cl::LocalSpaceArg, int
-                              >(*morKernels[device]);
+    kc_entry_t entry = kernelCache(device, refName);
+
+    if (entry.prog==0 && entry.ker==0) {
+        std::string options = generateOptionsString<T, isDilation, windLen>();
+        const char* ker_strs[] = {morph_cl};
+        const int   ker_lens[] = {morph_cl_len};
+        Program prog;
+        buildProgram(prog, 1, ker_strs, ker_lens, options);
+        entry.prog = new Program(prog);
+        entry.ker  = new Kernel(*entry.prog, "morph3d");
+        addKernelToCache(device, refName, entry);
+    }
+
+    auto morphOp = KernelFunctor< Buffer, KParam, Buffer, KParam, Buffer,
+                                  cl::LocalSpaceArg, int >(*entry.ker);
 
     NDRange local(CUBE_X, CUBE_Y, CUBE_Z);
 
@@ -146,9 +138,7 @@ void morph3d(Param       out,
     int blk_y = divup(in.info.dims[1], CUBE_Y);
     int blk_z = divup(in.info.dims[2], CUBE_Z);
     // launch batch * blk_x blocks along x dimension
-    NDRange global(blk_x * CUBE_X * in.info.dims[3],
-                    blk_y * CUBE_Y,
-                    blk_z * CUBE_Z);
+    NDRange global(blk_x * CUBE_X * in.info.dims[3], blk_y * CUBE_Y, blk_z * CUBE_Z);
 
     // copy mask/filter to constant memory
     cl_int se_size   = sizeof(T)*windLen*windLen*windLen;
@@ -169,7 +159,5 @@ void morph3d(Param       out,
     bufferFree(mBuff);
     CL_DEBUG_FINISH(getQueue());
 }
-
 }
-
 }
diff --git a/src/backend/opencl/kernel/orb.hpp b/src/backend/opencl/kernel/orb.hpp
index 5c312db5e1..cd3da4ed60 100644
--- a/src/backend/opencl/kernel/orb.hpp
+++ b/src/backend/opencl/kernel/orb.hpp
@@ -20,6 +20,7 @@
 #include <kernel_headers/orb.hpp>
 #include <memory.hpp>
 #include <vector>
+#include <cache.hpp>
 
 using cl::Buffer;
 using cl::Program;
@@ -50,10 +51,8 @@ using std::vector;
 
 namespace opencl
 {
-
 namespace kernel
 {
-
 static const int ORB_THREADS   = 256;
 static const int ORB_THREADS_X = 16;
 static const int ORB_THREADS_Y = 16;
@@ -86,50 +85,64 @@ void gaussian1D(T* out, const int dim, double sigma=0.0)
         out[k] /= sum;
 }
 
-template<typename T, typename convAccT>
-void orb(unsigned* out_feat,
-         Param& x_out,
-         Param& y_out,
-         Param& score_out,
-         Param& ori_out,
-         Param& size_out,
-         Param& desc_out,
-         Param image,
-         const float fast_thr,
-         const unsigned max_feat,
-         const float scl_fctr,
-         const unsigned levels,
-         const bool blur_img)
+template<typename T>
+std::tuple<cl::Kernel*, cl::Kernel*, cl::Kernel*, cl::Kernel*>
+getOrbKernels()
 {
-    static std::once_flag compileFlags[DeviceManager::MAX_DEVICES];
-    static std::map<int, Program*> orbProgs;
-    static std::map<int, Kernel*>  hrKernel;
-    static std::map<int, Kernel*>  kfKernel;
-    static std::map<int, Kernel*>  caKernel;
-    static std::map<int, Kernel*>  eoKernel;
+    static const std::string kernelNames[4] =
+        {"harris_response", "keep_features", "centroid_angle", "extract_orb"};
+
+    kc_entry_t entries[4];
 
     int device = getActiveDeviceId();
 
-    std::call_once( compileFlags[device], [device] () {
+    std::string checkName = kernelNames[0] + std::string("_") + std::string(dtype_traits<T>::getName());
 
-            std::ostringstream options;
-            options << " -D T=" << dtype_traits<T>::getName()
-                    << " -D BLOCK_SIZE=" << ORB_THREADS_X;
+    entries[0] = kernelCache(device, checkName);
 
-            if (std::is_same<T, double>::value ||
-                std::is_same<T, cdouble>::value) {
-                options << " -D USE_DOUBLE";
-            }
+    if (entries[0].prog==0 && entries[0].ker==0)
+    {
+        std::ostringstream options;
+        options << " -D T=" << dtype_traits<T>::getName()
+                << " -D BLOCK_SIZE=" << ORB_THREADS_X;
+
+        if (std::is_same<T, double>::value || std::is_same<T, cdouble>::value)
+            options << " -D USE_DOUBLE";
+
+        const char* ker_strs[] = {orb_cl};
+        const int   ker_lens[] = {orb_cl_len};
+        Program prog;
+        buildProgram(prog, 1, ker_strs, ker_lens, options.str());
+
+        for (int i=0; i<4; ++i)
+        {
+            entries[i].prog = new Program(prog);
+            entries[i].ker  = new Kernel(*entries[i].prog, kernelNames[i].c_str());
 
-            cl::Program prog;
-            buildProgram(prog, orb_cl, orb_cl_len, options.str());
-            orbProgs[device] = new Program(prog);
+            std::string name = kernelNames[i] +
+                std::string("_") + std::string(dtype_traits<T>::getName());
 
-            hrKernel[device] = new Kernel(*orbProgs[device], "harris_response");
-            kfKernel[device] = new Kernel(*orbProgs[device], "keep_features");
-            caKernel[device] = new Kernel(*orbProgs[device], "centroid_angle");
-            eoKernel[device] = new Kernel(*orbProgs[device], "extract_orb");
-        });
+            addKernelToCache(device, name, entries[i]);
+        }
+    } else {
+        for (int i=1; i<4; ++i) {
+            std::string name = kernelNames[i] +
+                std::string("_") + std::string(dtype_traits<T>::getName());
+
+            entries[i] = kernelCache(device, name);
+        }
+    }
+
+    return std::make_tuple(entries[0].ker, entries[1].ker, entries[2].ker, entries[3].ker);
+}
+
+template<typename T, typename convAccT>
+void orb(unsigned* out_feat, Param& x_out, Param& y_out, Param& score_out,
+         Param& ori_out, Param& size_out, Param& desc_out, Param image,
+         const float fast_thr, const unsigned max_feat, const float scl_fctr,
+         const unsigned levels, const bool blur_img)
+{
+    auto kernels = getOrbKernels<T>();
 
     unsigned patch_size = REF_PAT_SIZE;
 
@@ -253,7 +266,7 @@ void orb(unsigned* out_feat,
         auto hrOp = KernelFunctor<Buffer, Buffer, Buffer,
                                 Buffer, Buffer, const unsigned,
                                 Buffer, Buffer, KParam,
-                                const unsigned, const float, const unsigned> (*hrKernel[device]);
+                                const unsigned, const float, const unsigned> (*std::get<0>(kernels));
 
         hrOp(EnqueueArgs(getQueue(), global, local),
               *d_x_harris, *d_y_harris, *d_score_harris,
@@ -321,7 +334,7 @@ void orb(unsigned* out_feat,
 
         auto kfOp = KernelFunctor<Buffer, Buffer, Buffer,
                                 Buffer, Buffer, Buffer, Buffer,
-                                const unsigned> (*kfKernel[device]);
+                                const unsigned> (*std::get<1>(kernels));
 
         kfOp(EnqueueArgs(getQueue(), global_keep, local_keep),
               *d_x_lvl, *d_y_lvl, *d_score_lvl,
@@ -344,7 +357,7 @@ void orb(unsigned* out_feat,
 
         auto caOp = KernelFunctor<Buffer, Buffer, Buffer,
                                 const unsigned, Buffer, KParam,
-                                const unsigned> (*caKernel[device]);
+                                const unsigned> (*std::get<2>(kernels));
 
         caOp(EnqueueArgs(getQueue(), global_centroid, local_centroid),
               *d_x_lvl, *d_y_lvl, *d_ori_lvl,
@@ -396,7 +409,7 @@ void orb(unsigned* out_feat,
         auto eoOp = KernelFunctor<Buffer, const unsigned,
                                 Buffer, Buffer, Buffer, Buffer,
                                 Buffer, KParam,
-                                const float, const unsigned> (*eoKernel[device]);
+                                const float, const unsigned> (*std::get<3>(kernels));
 
         if (blur_img) {
             eoOp(EnqueueArgs(getQueue(), global_centroid, local_centroid),
@@ -514,9 +527,7 @@ void orb(unsigned* out_feat,
     // Sets number of output features
     *out_feat = total_feat;
 }
-
 } //namespace kernel
-
 } //namespace opencl
 
 #if defined(__clang__)
diff --git a/src/backend/opencl/kernel/range.hpp b/src/backend/opencl/kernel/range.hpp
index 0ea609f558..a1bb664b92 100644
--- a/src/backend/opencl/kernel/range.hpp
+++ b/src/backend/opencl/kernel/range.hpp
@@ -12,8 +12,7 @@
 #include <program.hpp>
 #include <traits.hpp>
 #include <string>
-#include <mutex>
-#include <map>
+#include <cache.hpp>
 #include <dispatch.hpp>
 #include <Param.hpp>
 #include <debug_opencl.hpp>
@@ -28,51 +27,51 @@ using std::string;
 
 namespace opencl
 {
-    namespace kernel
-    {
-        // Kernel Launch Config Values
-        static const int RANGE_TX = 32;
-        static const int RANGE_TY = 8;
-        static const int RANGE_TILEX = 512;
-        static const int RANGE_TILEY = 32;
+namespace kernel
+{
+// Kernel Launch Config Values
+static const int RANGE_TX = 32;
+static const int RANGE_TY = 8;
+static const int RANGE_TILEX = 512;
+static const int RANGE_TILEY = 32;
 
-        template<typename T>
-        void range(Param out, const int dim)
-        {
-            static std::once_flag compileFlags[DeviceManager::MAX_DEVICES];
-            static std::map<int, Program*>  rangeProgs;
-            static std::map<int, Kernel*> rangeKernels;
+template<typename T>
+void range(Param out, const int dim)
+{
+    std::string refName = std::string("range_kernel_") + std::string(dtype_traits<T>::getName());
 
-            int device = getActiveDeviceId();
+    int device = getActiveDeviceId();
+    kc_entry_t entry = kernelCache(device, refName);
 
-            std::call_once( compileFlags[device], [device] () {
-                std::ostringstream options;
-                options << " -D T=" << dtype_traits<T>::getName();
-                if (std::is_same<T, double>::value ||
-                    std::is_same<T, cdouble>::value) {
-                    options << " -D USE_DOUBLE";
-                }
-                Program prog;
-                buildProgram(prog, range_cl, range_cl_len, options.str());
-                rangeProgs[device]   = new Program(prog);
-                rangeKernels[device] = new Kernel(*rangeProgs[device], "range_kernel");
-            });
+    if (entry.prog==0 && entry.ker==0) {
+        std::ostringstream options;
+        options << " -D T=" << dtype_traits<T>::getName();
+        if (std::is_same<T, double>::value || std::is_same<T, cdouble>::value)
+            options << " -D USE_DOUBLE";
 
-            auto rangeOp = KernelFunctor<Buffer, const KParam, const int,
-                                        const int, const int> (*rangeKernels[device]);
+        const char* ker_strs[] = {range_cl};
+        const int   ker_lens[] = {range_cl_len};
+        Program prog;
+        buildProgram(prog, 1, ker_strs, ker_lens, options.str());
+        entry.prog = new Program(prog);
+        entry.ker  = new Kernel(*entry.prog, "range_kernel");
 
-            NDRange local(RANGE_TX, RANGE_TY, 1);
+        addKernelToCache(device, refName, entry);
+    }
 
-            int blocksPerMatX = divup(out.info.dims[0], RANGE_TILEX);
-            int blocksPerMatY = divup(out.info.dims[1], RANGE_TILEY);
-            NDRange global(local[0] * blocksPerMatX * out.info.dims[2],
-                            local[1] * blocksPerMatY * out.info.dims[3],
-                            1);
+    auto rangeOp = KernelFunctor< Buffer, const KParam, const int, const int, const int > (*entry.ker);
 
-            rangeOp(EnqueueArgs(getQueue(), global, local),
-                    *out.data, out.info, dim, blocksPerMatX, blocksPerMatY);
+    NDRange local(RANGE_TX, RANGE_TY, 1);
 
-            CL_DEBUG_FINISH(getQueue());
-        }
-    }
+    int blocksPerMatX = divup(out.info.dims[0], RANGE_TILEX);
+    int blocksPerMatY = divup(out.info.dims[1], RANGE_TILEY);
+    NDRange global(local[0] * blocksPerMatX * out.info.dims[2],
+                   local[1] * blocksPerMatY * out.info.dims[3], 1);
+
+    rangeOp(EnqueueArgs(getQueue(), global, local),
+            *out.data, out.info, dim, blocksPerMatX, blocksPerMatY);
+
+    CL_DEBUG_FINISH(getQueue());
+}
+}
 }
diff --git a/src/backend/opencl/kernel/regions.hpp b/src/backend/opencl/kernel/regions.hpp
index e381399342..35dd4bec52 100644
--- a/src/backend/opencl/kernel/regions.hpp
+++ b/src/backend/opencl/kernel/regions.hpp
@@ -16,7 +16,7 @@
 #include <debug_opencl.hpp>
 #include <math.hpp>
 #include <stdio.h>
-#include <map>
+#include <cache.hpp>
 #include <memory.hpp>
 
 #pragma GCC diagnostic push
@@ -43,57 +43,86 @@ namespace compute = boost::compute;
 
 namespace opencl
 {
-
 namespace kernel
 {
-
 static const int THREADS_X = 16;
 static const int THREADS_Y = 16;
 
 template<typename T, bool full_conn, int n_per_thread>
-void regions(Param out, Param in)
+std::tuple<cl::Kernel*, cl::Kernel*, cl::Kernel*>
+getRegionsKernels()
 {
-    static std::once_flag compileFlags[DeviceManager::MAX_DEVICES];
-    static std::map<int, Program*> regionsProgs;
-    static std::map<int, Kernel *>     ilKernel;
-    static std::map<int, Kernel *>     frKernel;
-    static std::map<int, Kernel *>     ueKernel;
-
-    int device = getActiveDeviceId();
     static const int block_dim = 16;
     static const int num_warps = 8;
+    static const unsigned NUM_KERNELS = 3;
+    static const std::string kernelNames[NUM_KERNELS] =
+        {"initial_label", "final_relabel", "update_equiv"};
 
-    std::call_once( compileFlags[device], [device] () {
-            ToNumStr<T> toNumStr;
-            std::ostringstream options;
-            if (full_conn) {
-                options << " -D T=" << dtype_traits<T>::getName()
-                        << " -D BLOCK_DIM=" << block_dim
-                        << " -D NUM_WARPS=" << num_warps
-                        << " -D N_PER_THREAD=" << n_per_thread
-                        << " -D LIMIT_MAX=" << toNumStr(maxval<T>())
-                        << " -D FULL_CONN";
-            }
-            else {
-                options << " -D T=" << dtype_traits<T>::getName()
-                        << " -D BLOCK_DIM=" << block_dim
-                        << " -D NUM_WARPS=" << num_warps
-                        << " -D N_PER_THREAD=" << n_per_thread
-                        << " -D LIMIT_MAX=" << toNumStr(maxval<T>());
-            }
-            if (std::is_same<T, double>::value ||
-                std::is_same<T, cdouble>::value) {
-                options << " -D USE_DOUBLE";
-            }
+    kc_entry_t entries[NUM_KERNELS];
+
+    int device = getActiveDeviceId();
+
+    std::string checkName = kernelNames[0] + std::string("_") +
+        std::string(dtype_traits<T>::getName()) +
+        std::to_string(full_conn) + std::to_string(n_per_thread);
+
+    entries[0] = kernelCache(device, checkName);
+
+    if (entries[0].prog==0 && entries[0].ker==0)
+    {
+        ToNumStr<T> toNumStr;
+        std::ostringstream options;
+        if (full_conn) {
+            options << " -D T=" << dtype_traits<T>::getName()
+                    << " -D BLOCK_DIM=" << block_dim
+                    << " -D NUM_WARPS=" << num_warps
+                    << " -D N_PER_THREAD=" << n_per_thread
+                    << " -D LIMIT_MAX=" << toNumStr(maxval<T>())
+                    << " -D FULL_CONN";
+        }
+        else {
+            options << " -D T=" << dtype_traits<T>::getName()
+                    << " -D BLOCK_DIM=" << block_dim
+                    << " -D NUM_WARPS=" << num_warps
+                    << " -D N_PER_THREAD=" << n_per_thread
+                    << " -D LIMIT_MAX=" << toNumStr(maxval<T>());
+        }
+        if (std::is_same<T, double>::value || std::is_same<T, cdouble>::value)
+            options << " -D USE_DOUBLE";
+
+        const char* ker_strs[] = {regions_cl};
+        const int   ker_lens[] = {regions_cl_len};
+        Program prog;
+        buildProgram(prog, 1, ker_strs, ker_lens, options.str());
 
-            Program prog;
-            buildProgram(prog, regions_cl, regions_cl_len, options.str());
-            regionsProgs[device] = new Program(prog);
+        for (unsigned i=0; i<NUM_KERNELS; ++i)
+        {
+            entries[i].prog = new Program(prog);
+            entries[i].ker  = new Kernel(*entries[i].prog, kernelNames[i].c_str());
+
+            std::string name = kernelNames[i] + std::string("_") +
+                std::string(dtype_traits<T>::getName()) +
+                std::to_string(full_conn) + std::to_string(n_per_thread);
 
-            ilKernel[device] = new Kernel(*regionsProgs[device], "initial_label");
-            frKernel[device] = new Kernel(*regionsProgs[device], "final_relabel");
-            ueKernel[device] = new Kernel(*regionsProgs[device], "update_equiv");
-        });
+            addKernelToCache(device, name, entries[i]);
+        }
+    } else {
+        for (unsigned i=1; i<NUM_KERNELS; ++i) {
+            std::string name = kernelNames[i] + std::string("_") +
+                std::string(dtype_traits<T>::getName()) +
+                std::to_string(full_conn) + std::to_string(n_per_thread);
+
+            entries[i] = kernelCache(device, name);
+        }
+    }
+
+    return std::make_tuple(entries[0].ker, entries[1].ker, entries[2].ker);
+}
+
+template<typename T, bool full_conn, int n_per_thread>
+void regions(Param out, Param in)
+{
+    auto kernels = getRegionsKernels<T, full_conn, n_per_thread>();
 
     const NDRange local(THREADS_X, THREADS_Y);
 
@@ -102,11 +131,9 @@ void regions(Param out, Param in)
 
     const NDRange global(blk_x * THREADS_X, blk_y * THREADS_Y);
 
-    auto ilOp = KernelFunctor<Buffer, KParam,
-                            Buffer, KParam> (*ilKernel[device]);
+    auto ilOp = KernelFunctor<Buffer, KParam, Buffer, KParam> (*std::get<0>(kernels));
 
-    ilOp(EnqueueArgs(getQueue(), global, local),
-          *out.data, out.info, *in.data, in.info);
+    ilOp(EnqueueArgs(getQueue(), global, local), *out.data, out.info, *in.data, in.info);
 
     CL_DEBUG_FINISH(getQueue());
 
@@ -117,8 +144,7 @@ void regions(Param out, Param in)
         h_continue = 0;
         getQueue().enqueueWriteBuffer(*d_continue, CL_TRUE, 0, sizeof(int), &h_continue);
 
-        auto ueOp = KernelFunctor<Buffer, KParam,
-                                Buffer> (*ueKernel[device]);
+        auto ueOp = KernelFunctor<Buffer, KParam, Buffer> (*std::get<2>(kernels));
 
         ueOp(EnqueueArgs(getQueue(), global, local),
               *out.data, out.info, *d_continue);
@@ -149,7 +175,8 @@ void regions(Param out, Param in)
     // compute.
     //int num_bins = tmp[size - 1] + 1;
     T last_label;
-    clEnqueueReadBuffer(getQueue()(), tmp.get_buffer().get(), CL_TRUE, (size - 1) * sizeof(T), sizeof(T), &last_label, 0, NULL, NULL);
+    clEnqueueReadBuffer(getQueue()(), tmp.get_buffer().get(), CL_TRUE,
+                        (size - 1) * sizeof(T), sizeof(T), &last_label, 0, NULL, NULL);
     int num_bins = (int)last_label + 1;
 
     Buffer labels(getContext(), CL_MEM_READ_WRITE, num_bins * sizeof(T));
@@ -176,7 +203,6 @@ void regions(Param out, Param in)
                 start = i + 1;
             }
         }
-
         return start;
     });
 
@@ -186,33 +212,22 @@ void regions(Param out, Param in)
     });
 
     compute::transform(search_begin, search_begin + num_bins,
-                        labels_begin,
-                        upper_bound_closure,
-                        c_queue);
+                       labels_begin, upper_bound_closure, c_queue);
     compute::adjacent_difference(labels_begin, labels_end, labels_begin, c_queue);
 
     // Perform the scan -- this can computes the correct labels for each
     // component
-    compute::transform(labels_begin, labels_end,
-                        labels_begin,
-                        clamp_to_one,
-                        c_queue);
-    compute::exclusive_scan(labels_begin,
-                            labels_end,
-                            labels_begin,
-                            c_queue);
+    compute::transform(labels_begin, labels_end, labels_begin, clamp_to_one, c_queue);
+
+    compute::exclusive_scan(labels_begin, labels_end, labels_begin, c_queue);
 
     // Apply the correct labels to the equivalency map
-    auto frOp = KernelFunctor<Buffer, KParam,
-                            Buffer, KParam,
-                            Buffer> (*frKernel[device]);
+    auto frOp = KernelFunctor<Buffer, KParam, Buffer, KParam, Buffer> (*std::get<1>(kernels));
 
     //Buffer labels_buf(tmp.get_buffer().get());
-    frOp(EnqueueArgs(getQueue(), global, local),
-          *out.data, out.info, *in.data, in.info, labels);
+    frOp(EnqueueArgs(getQueue(), global, local), *out.data, out.info, *in.data, in.info, labels);
+
     CL_DEBUG_FINISH(getQueue());
 }
-
 } //namespace kernel
-
 } //namespace opencl
diff --git a/src/backend/opencl/kernel/reorder.hpp b/src/backend/opencl/kernel/reorder.hpp
index 5bd7690e6d..87a9e725f3 100644
--- a/src/backend/opencl/kernel/reorder.hpp
+++ b/src/backend/opencl/kernel/reorder.hpp
@@ -12,8 +12,7 @@
 #include <program.hpp>
 #include <traits.hpp>
 #include <string>
-#include <mutex>
-#include <map>
+#include <cache.hpp>
 #include <dispatch.hpp>
 #include <Param.hpp>
 #include <debug_opencl.hpp>
@@ -28,54 +27,55 @@ using std::string;
 
 namespace opencl
 {
-    namespace kernel
-    {
-        // Kernel Launch Config Values
-        static const int TX = 32;
-        static const int TY = 8;
-        static const int TILEX = 512;
-        static const int TILEY = 32;
+namespace kernel
+{
+// Kernel Launch Config Values
+static const int TX = 32;
+static const int TY = 8;
+static const int TILEX = 512;
+static const int TILEY = 32;
 
-        template<typename T>
-        void reorder(Param out, const Param in, const dim_t *rdims)
-        {
-            static std::once_flag compileFlags[DeviceManager::MAX_DEVICES];
-            static std::map<int, Program*>   reorderProgs;
-            static std::map<int, Kernel *> reorderKernels;
+template<typename T>
+void reorder(Param out, const Param in, const dim_t *rdims)
+{
+    std::string refName = std::string("reorder_kernel_") + std::string(dtype_traits<T>::getName());
 
-            int device = getActiveDeviceId();
+    int device = getActiveDeviceId();
+    kc_entry_t entry = kernelCache(device, refName);
 
-            std::call_once( compileFlags[device], [device] () {
-                std::ostringstream options;
-                options << " -D T=" << dtype_traits<T>::getName();
-                if (std::is_same<T, double>::value ||
-                    std::is_same<T, cdouble>::value) {
-                    options << " -D USE_DOUBLE";
-                }
-                Program prog;
-                buildProgram(prog, reorder_cl, reorder_cl_len, options.str());
-                reorderProgs[device] = new Program(prog);
-                reorderKernels[device] = new Kernel(*reorderProgs[device], "reorder_kernel");
-            });
+    if (entry.prog==0 && entry.ker==0) {
+        std::ostringstream options;
+        options << " -D T=" << dtype_traits<T>::getName();
+        if (std::is_same<T, double>::value || std::is_same<T, cdouble>::value)
+            options << " -D USE_DOUBLE";
 
-            auto reorderOp = KernelFunctor<Buffer, const Buffer, const KParam, const KParam,
-                                      const int, const int, const int, const int,
-                                      const int, const int> (*reorderKernels[device]);
+        const char* ker_strs[] = {reorder_cl};
+        const int   ker_lens[] = {reorder_cl_len};
+        Program prog;
+        buildProgram(prog, 1, ker_strs, ker_lens, options.str());
+        entry.prog = new Program(prog);
+        entry.ker  = new Kernel(*entry.prog, "reorder_kernel");
 
-            NDRange local(TX, TY, 1);
+        addKernelToCache(device, refName, entry);
+    }
 
-            int blocksPerMatX = divup(out.info.dims[0], TILEX);
-            int blocksPerMatY = divup(out.info.dims[1], TILEY);
-            NDRange global(local[0] * blocksPerMatX * out.info.dims[2],
-                            local[1] * blocksPerMatY * out.info.dims[3],
-                            1);
+    auto reorderOp = KernelFunctor< Buffer, const Buffer, const KParam, const KParam,
+                                    const int, const int, const int, const int,
+                                    const int, const int >(*entry.ker);
 
-            reorderOp(EnqueueArgs(getQueue(), global, local),
-                      *out.data, *in.data, out.info, in.info,
-                      rdims[0], rdims[1], rdims[2], rdims[3],
-                      blocksPerMatX, blocksPerMatY);
+    NDRange local(TX, TY, 1);
 
-            CL_DEBUG_FINISH(getQueue());
-        }
-    }
+    int blocksPerMatX = divup(out.info.dims[0], TILEX);
+    int blocksPerMatY = divup(out.info.dims[1], TILEY);
+    NDRange global(local[0] * blocksPerMatX * out.info.dims[2],
+                   local[1] * blocksPerMatY * out.info.dims[3], 1);
+
+    reorderOp(EnqueueArgs(getQueue(), global, local),
+              *out.data, *in.data, out.info, in.info,
+              rdims[0], rdims[1], rdims[2], rdims[3],
+              blocksPerMatX, blocksPerMatY);
+
+    CL_DEBUG_FINISH(getQueue());
+}
+}
 }
diff --git a/src/backend/opencl/kernel/resize.hpp b/src/backend/opencl/kernel/resize.hpp
index bb4216c6a9..c58f343e5f 100644
--- a/src/backend/opencl/kernel/resize.hpp
+++ b/src/backend/opencl/kernel/resize.hpp
@@ -12,8 +12,7 @@
 #include <program.hpp>
 #include <traits.hpp>
 #include <string>
-#include <map>
-#include <mutex>
+#include <cache.hpp>
 #include <dispatch.hpp>
 #include <Param.hpp>
 #include <debug_opencl.hpp>
@@ -28,86 +27,84 @@ using std::string;
 
 namespace opencl
 {
-    namespace kernel
-    {
-        static const int RESIZE_TX = 16;
-        static const int RESIZE_TY = 16;
-
-        using std::conditional;
-        using std::is_same;
-        template<typename T>
-        using wtype_t = typename conditional<is_same<T, double>::value, double, float>::type;
-
-        template<typename T>
-        using vtype_t = typename conditional<is_complex<T>::value,
-                                             T, wtype_t<T>
-                                            >::type;
-
-        template<typename T, af_interp_type method>
-        void resize(Param out, const Param in)
-        {
-            static std::once_flag compileFlags[DeviceManager::MAX_DEVICES];
-            static std::map<int, Program*>   resizeProgs;
-            static std::map<int, Kernel *> resizeKernels;
-
-            int device = getActiveDeviceId();
-
-            typedef typename dtype_traits<T>::base_type BT;
-
-            std::call_once( compileFlags[device], [device] () {
-                std::ostringstream options;
-                options << " -D T="        << dtype_traits<T>::getName();
-                options << " -D VT="       << dtype_traits<vtype_t<T>>::getName();
-                options << " -D WT="       << dtype_traits<wtype_t<BT>>::getName();
-
-                switch(method) {
-                    case AF_INTERP_NEAREST:  options <<" -D INTERP=NEAREST" ; break;
-                    case AF_INTERP_BILINEAR: options <<" -D INTERP=BILINEAR"; break;
-                    case AF_INTERP_LOWER:    options <<" -D INTERP=LOWER"   ; break;
-                    default: break;
-                }
-
-                if((af_dtype) dtype_traits<T>::af_type == c32 ||
-                    (af_dtype) dtype_traits<T>::af_type == c64) {
-                    options << " -D CPLX=1";
-                    options << " -D TB=" << dtype_traits<BT>::getName();
-                } else {
-                    options << " -D CPLX=0";
-                }
-
-                if (std::is_same<T, double>::value ||
-                    std::is_same<T, cdouble>::value) {
-                    options << " -D USE_DOUBLE";
-                }
-
-                Program prog;
-                buildProgram(prog, resize_cl, resize_cl_len, options.str());
-                resizeProgs[device] = new Program(prog);
-                resizeKernels[device] = new Kernel(*resizeProgs[device], "resize_kernel");
-            });
-
-            auto resizeOp = KernelFunctor<Buffer, const KParam,
-                                  const Buffer, const KParam,
-                                  const int, const int, const float, const float>
-                                  (*resizeKernels[device]);
-
-            NDRange local(RESIZE_TX, RESIZE_TY, 1);
-
-            int blocksPerMatX = divup(out.info.dims[0], local[0]);
-            int blocksPerMatY = divup(out.info.dims[1], local[1]);
-            NDRange global(local[0] * blocksPerMatX * in.info.dims[2],
-                            local[1] * blocksPerMatY * in.info.dims[3],
-                            1);
-
-            double xd = (double)in.info.dims[0] / (double)out.info.dims[0];
-            double yd = (double)in.info.dims[1] / (double)out.info.dims[1];
-
-            float xf = (float)xd, yf = (float)yd;
-
-            resizeOp(EnqueueArgs(getQueue(), global, local),
-                      *out.data, out.info, *in.data, in.info, blocksPerMatX, blocksPerMatY, xf, yf);
-
-            CL_DEBUG_FINISH(getQueue());
+namespace kernel
+{
+static const int RESIZE_TX = 16;
+static const int RESIZE_TY = 16;
+
+using std::conditional;
+using std::is_same;
+template<typename T>
+using wtype_t = typename conditional<is_same<T, double>::value, double, float>::type;
+
+template<typename T>
+using vtype_t = typename conditional< is_complex<T>::value, T, wtype_t<T> >::type;
+
+template<typename T, af_interp_type method>
+void resize(Param out, const Param in)
+{
+    typedef typename dtype_traits<T>::base_type BT;
+
+    std::string refName = std::string("reorder_kernel_") +
+        std::string(dtype_traits<T>::getName()) +
+        std::to_string(method);
+
+    int device = getActiveDeviceId();
+    kc_entry_t entry = kernelCache(device, refName);
+
+    if (entry.prog==0 && entry.ker==0) {
+        std::ostringstream options;
+        options << " -D T="        << dtype_traits<T>::getName();
+        options << " -D VT="       << dtype_traits<vtype_t<T>>::getName();
+        options << " -D WT="       << dtype_traits<wtype_t<BT>>::getName();
+
+        switch(method) {
+            case AF_INTERP_NEAREST:  options <<" -D INTERP=NEAREST" ; break;
+            case AF_INTERP_BILINEAR: options <<" -D INTERP=BILINEAR"; break;
+            case AF_INTERP_LOWER:    options <<" -D INTERP=LOWER"   ; break;
+            default: break;
         }
+
+        if((af_dtype) dtype_traits<T>::af_type == c32 ||
+            (af_dtype) dtype_traits<T>::af_type == c64) {
+            options << " -D CPLX=1";
+            options << " -D TB=" << dtype_traits<BT>::getName();
+        } else {
+            options << " -D CPLX=0";
+        }
+
+        if (std::is_same<T, double>::value || std::is_same<T, cdouble>::value)
+            options << " -D USE_DOUBLE";
+
+        const char* ker_strs[] = {resize_cl};
+        const int   ker_lens[] = {resize_cl_len};
+        Program prog;
+        buildProgram(prog, 1, ker_strs, ker_lens, options.str());
+        entry.prog = new Program(prog);
+        entry.ker  = new Kernel(*entry.prog, "resize_kernel");
+
+        addKernelToCache(device, refName, entry);
     }
+
+    auto resizeOp = KernelFunctor< Buffer, const KParam, const Buffer, const KParam,
+                                   const int, const int, const float, const float > (*entry.ker);
+
+    NDRange local(RESIZE_TX, RESIZE_TY, 1);
+
+    int blocksPerMatX = divup(out.info.dims[0], local[0]);
+    int blocksPerMatY = divup(out.info.dims[1], local[1]);
+    NDRange global(local[0] * blocksPerMatX * in.info.dims[2],
+                   local[1] * blocksPerMatY * in.info.dims[3], 1);
+
+    double xd = (double)in.info.dims[0] / (double)out.info.dims[0];
+    double yd = (double)in.info.dims[1] / (double)out.info.dims[1];
+
+    float xf = (float)xd, yf = (float)yd;
+
+    resizeOp(EnqueueArgs(getQueue(), global, local),
+             *out.data, out.info, *in.data, in.info, blocksPerMatX, blocksPerMatY, xf, yf);
+
+    CL_DEBUG_FINISH(getQueue());
+}
+}
 }
diff --git a/src/backend/opencl/kernel/rotate.hpp b/src/backend/opencl/kernel/rotate.hpp
index 521383524d..4f595b5ec4 100644
--- a/src/backend/opencl/kernel/rotate.hpp
+++ b/src/backend/opencl/kernel/rotate.hpp
@@ -13,8 +13,7 @@
 #include <program.hpp>
 #include <traits.hpp>
 #include <string>
-#include <mutex>
-#include <map>
+#include <cache.hpp>
 #include <dispatch.hpp>
 #include <Param.hpp>
 #include <debug_opencl.hpp>
@@ -33,122 +32,119 @@ using std::string;
 
 namespace opencl
 {
-    namespace kernel
-    {
-        static const int TX = 16;
-        static const int TY = 16;
-        // Used for batching images
-        static const int TI = 4;
-
-        typedef struct {
-            float tmat[6];
-        } tmat_t;
-
-        using std::conditional;
-        using std::is_same;
-        template<typename T>
-        using wtype_t = typename conditional<is_same<T, double>::value, double, float>::type;
-
-        template<typename T>
-        using vtype_t = typename conditional<is_complex<T>::value,
-                                             T, wtype_t<T>
-                                            >::type;
-
-        template<typename T, int order>
-        void rotate(Param out, const Param in, const float theta, af_interp_type method)
-        {
-            static std::once_flag compileFlags[DeviceManager::MAX_DEVICES];
-            static std::map<int, Program*>   rotateProgs;
-            static std::map<int, Kernel *> rotateKernels;
-
-            int device = getActiveDeviceId();
-            typedef typename dtype_traits<T>::base_type BT;
-
-            std::call_once( compileFlags[device], [device] () {
-                ToNumStr<T> toNumStr;
-                std::ostringstream options;
-                options << " -D T="        << dtype_traits<T>::getName();
-                options << " -D ZERO="      << toNumStr(scalar<T>(0));
-                options << " -D InterpInTy=" << dtype_traits<T>::getName();
-                options << " -D InterpValTy="  << dtype_traits<vtype_t<T>>::getName();
-                options << " -D InterpPosTy=" << dtype_traits<wtype_t<BT>>::getName();
-
-                if((af_dtype) dtype_traits<T>::af_type == c32 ||
-                    (af_dtype) dtype_traits<T>::af_type == c64) {
-                    options << " -D IS_CPLX=1";
-                    options << " -D TB=" << dtype_traits<BT>::getName();
-                } else {
-                    options << " -D IS_CPLX=0";
-                }
-                if (std::is_same<T, double>::value ||
-                    std::is_same<T, cdouble>::value) {
-                    options << " -D USE_DOUBLE";
-                }
-
-                options << " -D INTERP_ORDER=" << order;
-                addInterpEnumOptions(options);
-
-                const char *ker_strs[] = {interp_cl, rotate_cl};
-                const int   ker_lens[] = {interp_cl_len, rotate_cl_len};
-                Program prog;
-                buildProgram(prog, 2, ker_strs, ker_lens, options.str());
-                rotateProgs[device] = new Program(prog);
-                rotateKernels[device] = new Kernel(*rotateProgs[device], "rotate_kernel");
-            });
-
-            auto rotateOp = KernelFunctor<Buffer, const KParam,
-                                          const Buffer, const KParam,
-                                          const tmat_t,
-                                          const int, const int,
-                                          const int, const int,
-                                          const int>(*rotateKernels[device]);
-
-            const float c = cos(-theta), s = sin(-theta);
-            float tx, ty;
-            {
-                const float nx = 0.5 * (in.info.dims[0] - 1);
-                const float ny = 0.5 * (in.info.dims[1] - 1);
-                const float mx = 0.5 * (out.info.dims[0] - 1);
-                const float my = 0.5 * (out.info.dims[1] - 1);
-                const float sx = (mx * c + my *-s);
-                const float sy = (mx * s + my * c);
-                tx = -(sx - nx);
-                ty = -(sy - ny);
-            }
-
-            // Rounding error. Anything more than 3 decimal points wont make a diff
-            tmat_t t;
-            t.tmat[0] = round( c * 1000) / 1000.0f;
-            t.tmat[1] = round(-s * 1000) / 1000.0f;
-            t.tmat[2] = round(tx * 1000) / 1000.0f;
-            t.tmat[3] = round( s * 1000) / 1000.0f;
-            t.tmat[4] = round( c * 1000) / 1000.0f;
-            t.tmat[5] = round(ty * 1000) / 1000.0f;
-
-
-            NDRange local(TX, TY, 1);
-
-            int nimages  = in.info.dims[2];
-            int nbatches = in.info.dims[3];
-            int global_x = local[0] * divup(out.info.dims[0], local[0]);
-            int global_y = local[1] * divup(out.info.dims[1], local[1]);
-            const int blocksXPerImage = global_x / local[0];
-            const int blocksYPerImage = global_y / local[1];
-
-            if(nimages > TI) {
-                int tile_images = divup(nimages, TI);
-                nimages = TI;
-                global_x = global_x * tile_images;
-            }
-            global_y *= nbatches;
-
-            NDRange global(global_x, global_y, 1);
-
-            rotateOp(EnqueueArgs(getQueue(), global, local),
-                      *out.data, out.info, *in.data, in.info, t, nimages, nbatches,
-                      blocksXPerImage, blocksYPerImage, (int)method);
-
-            CL_DEBUG_FINISH(getQueue());
+namespace kernel
+{
+static const int TX = 16;
+static const int TY = 16;
+// Used for batching images
+static const int TI = 4;
+
+typedef struct {
+    float tmat[6];
+} tmat_t;
+
+using std::conditional;
+using std::is_same;
+template<typename T>
+using wtype_t = typename conditional<is_same<T, double>::value, double, float>::type;
+
+template<typename T>
+using vtype_t = typename conditional< is_complex<T>::value, T, wtype_t<T> >::type;
+
+template<typename T, int order>
+void rotate(Param out, const Param in, const float theta, af_interp_type method)
+{
+    typedef typename dtype_traits<T>::base_type BT;
+
+    std::string refName = std::string("rotate_kernel_") +
+        std::string(dtype_traits<T>::getName()) +
+        std::to_string(order);
+
+    int device = getActiveDeviceId();
+    kc_entry_t entry = kernelCache(device, refName);
+
+    if (entry.prog==0 && entry.ker==0) {
+        ToNumStr<T> toNumStr;
+        std::ostringstream options;
+        options << " -D T="        << dtype_traits<T>::getName();
+        options << " -D ZERO="      << toNumStr(scalar<T>(0));
+        options << " -D InterpInTy=" << dtype_traits<T>::getName();
+        options << " -D InterpValTy="  << dtype_traits<vtype_t<T>>::getName();
+        options << " -D InterpPosTy=" << dtype_traits<wtype_t<BT>>::getName();
+
+        if((af_dtype) dtype_traits<T>::af_type == c32 ||
+            (af_dtype) dtype_traits<T>::af_type == c64) {
+            options << " -D IS_CPLX=1";
+            options << " -D TB=" << dtype_traits<BT>::getName();
+        } else {
+            options << " -D IS_CPLX=0";
         }
+        if (std::is_same<T, double>::value || std::is_same<T, cdouble>::value)
+            options << " -D USE_DOUBLE";
+
+        options << " -D INTERP_ORDER=" << order;
+        addInterpEnumOptions(options);
+
+        const char *ker_strs[] = {interp_cl, rotate_cl};
+        const int   ker_lens[] = {interp_cl_len, rotate_cl_len};
+        Program prog;
+        buildProgram(prog, 2, ker_strs, ker_lens, options.str());
+        entry.prog = new Program(prog);
+        entry.ker  = new Kernel(*entry.prog, "rotate_kernel");
+
+        addKernelToCache(device, refName, entry);
     }
+
+    auto rotateOp = KernelFunctor<Buffer, const KParam, const Buffer, const KParam,
+                                  const tmat_t, const int, const int,
+                                  const int, const int, const int>(*entry.ker);
+
+    const float c = cos(-theta), s = sin(-theta);
+    float tx, ty;
+    {
+        const float nx = 0.5 * (in.info.dims[0] - 1);
+        const float ny = 0.5 * (in.info.dims[1] - 1);
+        const float mx = 0.5 * (out.info.dims[0] - 1);
+        const float my = 0.5 * (out.info.dims[1] - 1);
+        const float sx = (mx * c + my *-s);
+        const float sy = (mx * s + my * c);
+        tx = -(sx - nx);
+        ty = -(sy - ny);
+    }
+
+    // Rounding error. Anything more than 3 decimal points wont make a diff
+    tmat_t t;
+    t.tmat[0] = round( c * 1000) / 1000.0f;
+    t.tmat[1] = round(-s * 1000) / 1000.0f;
+    t.tmat[2] = round(tx * 1000) / 1000.0f;
+    t.tmat[3] = round( s * 1000) / 1000.0f;
+    t.tmat[4] = round( c * 1000) / 1000.0f;
+    t.tmat[5] = round(ty * 1000) / 1000.0f;
+
+
+    NDRange local(TX, TY, 1);
+
+    int nimages  = in.info.dims[2];
+    int nbatches = in.info.dims[3];
+    int global_x = local[0] * divup(out.info.dims[0], local[0]);
+    int global_y = local[1] * divup(out.info.dims[1], local[1]);
+    const int blocksXPerImage = global_x / local[0];
+    const int blocksYPerImage = global_y / local[1];
+
+    if(nimages > TI) {
+        int tile_images = divup(nimages, TI);
+        nimages = TI;
+        global_x = global_x * tile_images;
+    }
+    global_y *= nbatches;
+
+    NDRange global(global_x, global_y, 1);
+
+    rotateOp(EnqueueArgs(getQueue(), global, local),
+             *out.data, out.info, *in.data, in.info, t, nimages, nbatches,
+             blocksXPerImage, blocksYPerImage, (int)method);
+
+    CL_DEBUG_FINISH(getQueue());
+}
+}
 }
diff --git a/src/backend/opencl/kernel/select.hpp b/src/backend/opencl/kernel/select.hpp
index 12d55fa60a..ed8fe02640 100644
--- a/src/backend/opencl/kernel/select.hpp
+++ b/src/backend/opencl/kernel/select.hpp
@@ -12,8 +12,7 @@
 #include <program.hpp>
 #include <traits.hpp>
 #include <string>
-#include <mutex>
-#include <map>
+#include <cache.hpp>
 #include <dispatch.hpp>
 #include <Param.hpp>
 #include <debug_opencl.hpp>
@@ -30,143 +29,120 @@ using std::string;
 
 namespace opencl
 {
-    namespace kernel
-    {
-        static const uint DIMX = 32;
-        static const uint DIMY =  8;
-        static const int REPEAT = 64;
-
-        template<typename T, bool is_same>
-        void select_launcher(Param out, Param cond, Param a, Param b, int ndims)
-        {
-            static std::once_flag compileFlags[DeviceManager::MAX_DEVICES];
-            static std::map<int, Program*>  selProgs;
-            static std::map<int, Kernel*> selKernels;
-
-            int device = getActiveDeviceId();
-
-            std::call_once(compileFlags[device], [device] () {
-
-                    std::ostringstream options;
-                    options << " -D is_same=" << is_same
-                            << " -D T=" << dtype_traits<T>::getName();
-
-                    if (std::is_same<T, double>::value ||
-                        std::is_same<T, cdouble>::value) {
-                        options << " -D USE_DOUBLE";
-                    }
-
-                    cl::Program prog;
-                    buildProgram(prog, select_cl, select_cl_len, options.str());
-                    selProgs[device] = new Program(prog);
-
-                    selKernels[device] = new Kernel(*selProgs[device], "select_kernel");
-                });
-
-
-            int threads[] = {DIMX, DIMY};
-
-            if (ndims == 1) {
-                threads[0] *= threads[1];
-                threads[1] = 1;
-            }
-
-            NDRange local(threads[0],
-                          threads[1]);
-
-
-            int groups_0 = divup(out.info.dims[0], REPEAT * local[0]);
-            int groups_1 = divup(out.info.dims[1], local[1]);
-
-            NDRange global(groups_0 * out.info.dims[2] * local[0],
-                           groups_1 * out.info.dims[3] * local[1]);
-
-            auto selectOp = KernelFunctor<Buffer, KParam,
-                                        Buffer, KParam,
-                                        Buffer, KParam,
-                                        Buffer, KParam,
-                                        int, int>(*selKernels[device]);
-
-            selectOp(EnqueueArgs(getQueue(), global, local),
-                     *out.data, out.info,
-                     *cond.data, cond.info,
-                     *a.data, a.info,
-                     *b.data, b.info,
-                     groups_0, groups_1);
+namespace kernel
+{
+static const uint DIMX = 32;
+static const uint DIMY =  8;
+static const int REPEAT = 64;
 
-        }
+template<typename T, bool is_same>
+void select_launcher(Param out, Param cond, Param a, Param b, int ndims)
+{
+    std::string refName = std::string("select_kernel_") +
+        std::string(dtype_traits<T>::getName()) + std::to_string(is_same);
+
+    int device = getActiveDeviceId();
+    kc_entry_t entry = kernelCache(device, refName);
+
+    if (entry.prog==0 && entry.ker==0) {
+        std::ostringstream options;
+        options << " -D is_same=" << is_same << " -D T=" << dtype_traits<T>::getName();
+        if (std::is_same<T, double>::value || std::is_same<T, cdouble>::value)
+            options << " -D USE_DOUBLE";
+
+        const char* ker_strs[] = {select_cl};
+        const int   ker_lens[] = {select_cl_len};
+        Program prog;
+        buildProgram(prog, 1, ker_strs, ker_lens, options.str());
+        entry.prog = new Program(prog);
+        entry.ker  = new Kernel(*entry.prog, "select_kernel");
+
+        addKernelToCache(device, refName, entry);
+    }
 
-        template<typename T>
-        void select(Param out, Param cond, Param a, Param b, int ndims)
-        {
-            bool is_same = true;
-            for (int i = 0; i < 4; i++) {
-                is_same &= (a.info.dims[i] == b.info.dims[i]);
-            }
+    int threads[] = {DIMX, DIMY};
 
-            if (is_same) {
-                select_launcher<T, true >(out, cond, a, b, ndims);
-            } else {
-                select_launcher<T, false>(out, cond, a, b, ndims);
-            }
-        }
+    if (ndims == 1) {
+        threads[0] *= threads[1];
+        threads[1] = 1;
+    }
 
-        template<typename T, bool flip>
-        void select_scalar(Param out, Param cond, Param a, const double b, int ndims)
-        {
-            static std::once_flag compileFlags[DeviceManager::MAX_DEVICES];
-            static std::map<int, Program*>  selProgs;
-            static std::map<int, Kernel*> selKernels;
+    NDRange local(threads[0], threads[1]);
 
-            int device = getActiveDeviceId();
+    int groups_0 = divup(out.info.dims[0], REPEAT * local[0]);
+    int groups_1 = divup(out.info.dims[1], local[1]);
 
-            std::call_once(compileFlags[device], [device] () {
+    NDRange global(groups_0 * out.info.dims[2] * local[0], groups_1 * out.info.dims[3] * local[1]);
 
-                    std::ostringstream options;
-                    options << " -D flip=" << flip
-                            << " -D T=" << dtype_traits<T>::getName();
+    auto selectOp = KernelFunctor< Buffer, KParam, Buffer, KParam, Buffer, KParam,
+                                   Buffer, KParam, int, int>(*entry.ker);
 
-                    if (std::is_same<T, double>::value ||
-                        std::is_same<T, cdouble>::value) {
-                        options << " -D USE_DOUBLE";
-                    }
+    selectOp(EnqueueArgs(getQueue(), global, local),
+             *out.data, out.info, *cond.data, cond.info, *a.data, a.info,
+             *b.data, b.info, groups_0, groups_1);
+}
 
-                    cl::Program prog;
-                    buildProgram(prog, select_cl, select_cl_len, options.str());
-                    selProgs[device] = new Program(prog);
+template<typename T>
+void select(Param out, Param cond, Param a, Param b, int ndims)
+{
+    bool is_same = true;
+    for (int i = 0; i < 4; i++) {
+        is_same &= (a.info.dims[i] == b.info.dims[i]);
+    }
 
-                    selKernels[device] = new Kernel(*selProgs[device], "select_scalar_kernel");
-                });
+    if (is_same) {
+        select_launcher<T, true >(out, cond, a, b, ndims);
+    } else {
+        select_launcher<T, false>(out, cond, a, b, ndims);
+    }
+}
 
+template<typename T, bool flip>
+void select_scalar(Param out, Param cond, Param a, const double b, int ndims)
+{
+    std::string refName = std::string("select_scalar_kernel_") +
+        std::string(dtype_traits<T>::getName()) +
+        std::to_string(flip);
+
+    int device = getActiveDeviceId();
+    kc_entry_t entry = kernelCache(device, refName);
+
+    if (entry.prog==0 && entry.ker==0) {
+        std::ostringstream options;
+        options << " -D flip=" << flip << " -D T=" << dtype_traits<T>::getName();
+        if (std::is_same<T, double>::value || std::is_same<T, cdouble>::value)
+            options << " -D USE_DOUBLE";
+
+        const char* ker_strs[] = {select_cl};
+        const int   ker_lens[] = {select_cl_len};
+        Program prog;
+        buildProgram(prog, 1, ker_strs, ker_lens, options.str());
+        entry.prog = new Program(prog);
+        entry.ker  = new Kernel(*entry.prog, "select_scalar_kernel");
+
+        addKernelToCache(device, refName, entry);
+    }
 
-            int threads[] = {DIMX, DIMY};
+    int threads[] = {DIMX, DIMY};
 
-            if (ndims == 1) {
-                threads[0] *= threads[1];
-                threads[1] = 1;
-            }
+    if (ndims == 1) {
+        threads[0] *= threads[1];
+        threads[1] = 1;
+    }
 
-            NDRange local(threads[0],
-                          threads[1]);
+    NDRange local(threads[0], threads[1]);
 
-            int groups_0 = divup(out.info.dims[0], REPEAT * local[0]);
-            int groups_1 = divup(out.info.dims[1], local[1]);
+    int groups_0 = divup(out.info.dims[0], REPEAT * local[0]);
+    int groups_1 = divup(out.info.dims[1], local[1]);
 
-            NDRange global(groups_0 * out.info.dims[2] * local[0],
-                           groups_1 * out.info.dims[3] * local[1]);
+    NDRange global(groups_0 * out.info.dims[2] * local[0], groups_1 * out.info.dims[3] * local[1]);
 
-            auto selectOp = KernelFunctor<Buffer, KParam,
-                                        Buffer, KParam,
-                                        Buffer, KParam,
-                                        T,
-                                        int, int>(*selKernels[device]);
+    auto selectOp = KernelFunctor< Buffer, KParam, Buffer, KParam, Buffer, KParam,
+                                   T, int, int>(*entry.ker);
 
-            selectOp(EnqueueArgs(getQueue(), global, local),
-                     *out.data, out.info,
-                     *cond.data, cond.info,
-                     *a.data, a.info,
-                     scalar<T>(b),
-                     groups_0, groups_1);
-        }
-    }
+    selectOp(EnqueueArgs(getQueue(), global, local),
+             *out.data, out.info, *cond.data, cond.info,
+             *a.data, a.info, scalar<T>(b), groups_0, groups_1);
+}
+}
 }
diff --git a/src/backend/opencl/kernel/shift.hpp b/src/backend/opencl/kernel/shift.hpp
index 1bbfbe9fdc..7a6fb81650 100644
--- a/src/backend/opencl/kernel/shift.hpp
+++ b/src/backend/opencl/kernel/shift.hpp
@@ -12,8 +12,7 @@
 #include <program.hpp>
 #include <traits.hpp>
 #include <string>
-#include <mutex>
-#include <map>
+#include <cache.hpp>
 #include <dispatch.hpp>
 #include <Param.hpp>
 #include <debug_opencl.hpp>
@@ -29,63 +28,64 @@ using std::string;
 
 namespace opencl
 {
-    namespace kernel
-    {
-        // Kernel Launch Config Values
-        static const int TX = 32;
-        static const int TY = 8;
-        static const int TILEX = 128;
-        static const int TILEY = 32;
+namespace kernel
+{
+// Kernel Launch Config Values
+static const int TX = 32;
+static const int TY = 8;
+static const int TILEX = 128;
+static const int TILEY = 32;
 
-        template<typename T>
-        void shift(Param out, const Param in, const int *sdims)
-        {
-            static std::once_flag compileFlags[DeviceManager::MAX_DEVICES];
-            static std::map<int, Program*>   shiftProgs;
-            static std::map<int, Kernel *> shiftKernels;
+template<typename T>
+void shift(Param out, const Param in, const int *sdims)
+{
+    std::string refName = std::string("shift_kernel_") + std::string(dtype_traits<T>::getName());
 
-            int device = getActiveDeviceId();
+    int device = getActiveDeviceId();
+    kc_entry_t entry = kernelCache(device, refName);
 
-            std::call_once( compileFlags[device], [device] () {
-                std::ostringstream options;
-                options << " -D T=" << dtype_traits<T>::getName();
-                if (std::is_same<T, double>::value ||
-                    std::is_same<T, cdouble>::value) {
-                    options << " -D USE_DOUBLE";
-                }
-                Program prog;
-                buildProgram(prog, shift_cl, shift_cl_len, options.str());
-                shiftProgs[device] = new Program(prog);
-                shiftKernels[device] = new Kernel(*shiftProgs[device], "shift_kernel");
-            });
+    if (entry.prog==0 && entry.ker==0) {
+        std::ostringstream options;
+        options << " -D T=" << dtype_traits<T>::getName();
+        if (std::is_same<T, double>::value || std::is_same<T, cdouble>::value)
+            options << " -D USE_DOUBLE";
 
-            auto shiftOp = KernelFunctor<Buffer, const Buffer, const KParam, const KParam,
-                                      const int, const int, const int, const int,
-                                      const int, const int> (*shiftKernels[device]);
+        const char* ker_strs[] = {shift_cl};
+        const int   ker_lens[] = {shift_cl_len};
+        Program prog;
+        buildProgram(prog, 1, ker_strs, ker_lens, options.str());
+        entry.prog = new Program(prog);
+        entry.ker  = new Kernel(*entry.prog, "shift_kernel");
 
-            NDRange local(TX, TY, 1);
+        addKernelToCache(device, refName, entry);
+    }
 
-            int blocksPerMatX = divup(out.info.dims[0], TILEX);
-            int blocksPerMatY = divup(out.info.dims[1], TILEY);
-            NDRange global(local[0] * blocksPerMatX * out.info.dims[2],
-                            local[1] * blocksPerMatY * out.info.dims[3],
-                            1);
+    auto shiftOp = KernelFunctor< Buffer, const Buffer, const KParam, const KParam,
+                                  const int, const int, const int, const int,
+                                  const int, const int> (*entry.ker);
 
-            int sdims_[4];
-            // Need to do this because we are mapping output to input in the kernel
-            for(int i = 0; i < 4; i++) {
-                // sdims_[i] will always be positive and always [0, oDims[i]].
-                // Negative shifts are converted to position by going the other way round
-                sdims_[i] = -(sdims[i] % (int)out.info.dims[i]) + out.info.dims[i] * (sdims[i] > 0);
-                assert(sdims_[i] >= 0 && sdims_[i] <= out.info.dims[i]);
-            }
+    NDRange local(TX, TY, 1);
 
-            shiftOp(EnqueueArgs(getQueue(), global, local),
-                    *out.data, *in.data, out.info, in.info,
-                    sdims_[0], sdims_[1], sdims_[2], sdims_[3],
-                    blocksPerMatX, blocksPerMatY);
+    int blocksPerMatX = divup(out.info.dims[0], TILEX);
+    int blocksPerMatY = divup(out.info.dims[1], TILEY);
+    NDRange global(local[0] * blocksPerMatX * out.info.dims[2],
+                   local[1] * blocksPerMatY * out.info.dims[3], 1);
 
-            CL_DEBUG_FINISH(getQueue());
-        }
+    int sdims_[4];
+    // Need to do this because we are mapping output to input in the kernel
+    for(int i = 0; i < 4; i++) {
+        // sdims_[i] will always be positive and always [0, oDims[i]].
+        // Negative shifts are converted to position by going the other way round
+        sdims_[i] = -(sdims[i] % (int)out.info.dims[i]) + out.info.dims[i] * (sdims[i] > 0);
+        assert(sdims_[i] >= 0 && sdims_[i] <= out.info.dims[i]);
     }
+
+    shiftOp(EnqueueArgs(getQueue(), global, local),
+            *out.data, *in.data, out.info, in.info,
+            sdims_[0], sdims_[1], sdims_[2], sdims_[3],
+            blocksPerMatX, blocksPerMatY);
+
+    CL_DEBUG_FINISH(getQueue());
+}
+}
 }
diff --git a/src/backend/opencl/kernel/sift_nonfree.hpp b/src/backend/opencl/kernel/sift_nonfree.hpp
index af01b60d45..229ba64560 100644
--- a/src/backend/opencl/kernel/sift_nonfree.hpp
+++ b/src/backend/opencl/kernel/sift_nonfree.hpp
@@ -93,6 +93,7 @@
 #include <kernel/resize.hpp>
 #include <kernel_headers/sift_nonfree.hpp>
 #include <memory.hpp>
+#include <cache.hpp>
 #include <vector>
 
 namespace compute = boost::compute;
@@ -107,10 +108,8 @@ using std::vector;
 
 namespace opencl
 {
-
 namespace kernel
 {
-
 static const int SIFT_THREADS   = 256;
 static const int SIFT_THREADS_X = 32;
 static const int SIFT_THREADS_Y = 8;
@@ -396,59 +395,66 @@ void apply_permutation(compute::buffer_iterator<T>& keys, compute::vector<int>&
     compute::gather(permutation.begin(), permutation.end(), temp.begin(), keys, queue);
 }
 
-template<typename T, typename convAccT>
-void sift(unsigned* out_feat,
-          unsigned* out_dlen,
-          Param& x_out,
-          Param& y_out,
-          Param& score_out,
-          Param& ori_out,
-          Param& size_out,
-          Param& desc_out,
-          Param img,
-          const unsigned n_layers,
-          const float contrast_thr,
-          const float edge_thr,
-          const float init_sigma,
-          const bool double_input,
-          const float img_scale,
-          const float feature_ratio,
-          const bool compute_GLOH)
+template<typename T>
+std::array<cl::Kernel*, 7> getSiftKernels()
 {
-    static std::once_flag compileFlags[DeviceManager::MAX_DEVICES];
-    static std::map<int, Program*> siftProgs;
-    static std::map<int, Kernel*>  suKernel;
-    static std::map<int, Kernel*>  deKernel;
-    static std::map<int, Kernel*>  ieKernel;
-    static std::map<int, Kernel*>  coKernel;
-    static std::map<int, Kernel*>  rdKernel;
-    static std::map<int, Kernel*>  cdKernel;
-    static std::map<int, Kernel*>  cgKernel;
+    static const unsigned NUM_KERNELS = 7;
+    static const std::string kernelNames[NUM_KERNELS] =
+        {"sub", "detectExtrema", "interpolateExtrema", "calcOrientation", "removeDuplicates",
+         "computeDescriptor", "computeGLOHDescriptor"};
+
+    kc_entry_t entries[NUM_KERNELS];
 
     int device = getActiveDeviceId();
 
-    std::call_once( compileFlags[device], [device] () {
+    std::string checkName = kernelNames[0] + std::string("_") + std::string(dtype_traits<T>::getName());
 
-            std::ostringstream options;
-            options << " -D T=" << dtype_traits<T>::getName();
+    entries[0] = kernelCache(device, checkName);
 
-            if (std::is_same<T, double>::value ||
-                std::is_same<T, cdouble>::value) {
-                options << " -D USE_DOUBLE";
-            }
+    if (entries[0].prog==0 && entries[0].ker==0)
+    {
+        std::ostringstream options;
+        options << " -D T=" << dtype_traits<T>::getName();
+        if (std::is_same<T, double>::value || std::is_same<T, cdouble>::value)
+            options << " -D USE_DOUBLE";
 
-            cl::Program prog;
-            buildProgram(prog, sift_nonfree_cl, sift_nonfree_cl_len, options.str());
-            siftProgs[device] = new Program(prog);
+        cl::Program prog;
+        buildProgram(prog, sift_nonfree_cl, sift_nonfree_cl_len, options.str());
 
-            suKernel[device] = new Kernel(*siftProgs[device], "sub");
-            deKernel[device] = new Kernel(*siftProgs[device], "detectExtrema");
-            ieKernel[device] = new Kernel(*siftProgs[device], "interpolateExtrema");
-            coKernel[device] = new Kernel(*siftProgs[device], "calcOrientation");
-            rdKernel[device] = new Kernel(*siftProgs[device], "removeDuplicates");
-            cdKernel[device] = new Kernel(*siftProgs[device], "computeDescriptor");
-            cgKernel[device] = new Kernel(*siftProgs[device], "computeGLOHDescriptor");
-        });
+        for (unsigned i=0; i<NUM_KERNELS; ++i)
+        {
+            entries[i].prog = new Program(prog);
+            entries[i].ker  = new Kernel(*entries[i].prog, kernelNames[i].c_str());
+
+            std::string name = kernelNames[i] + std::string("_") +
+                std::string(dtype_traits<T>::getName());
+
+            addKernelToCache(device, name, entries[i]);
+        }
+    } else {
+        for (unsigned i=1; i<NUM_KERNELS; ++i) {
+            std::string name = kernelNames[i] + std::string("_") +
+                std::string(dtype_traits<T>::getName());
+
+            entries[i] = kernelCache(device, name);
+        }
+    }
+
+    std::array<cl::Kernel*, NUM_KERNELS> retVal;
+    for (unsigned i=0; i<NUM_KERNELS; ++i)
+        retVal[i] = entries[i].ker;
+
+    return retVal;
+}
+
+template<typename T, typename convAccT>
+void sift(unsigned* out_feat, unsigned* out_dlen, Param& x_out, Param& y_out,
+          Param& score_out, Param& ori_out, Param& size_out, Param& desc_out,
+          Param img, const unsigned n_layers, const float contrast_thr, const float edge_thr,
+          const float init_sigma, const bool double_input, const float img_scale,
+          const float feature_ratio, const bool compute_GLOH)
+{
+    auto kernels = getSiftKernels<T>();
 
     const unsigned min_dim = (double_input) ? min(img.info.dims[0]*2, img.info.dims[1]*2)
                                             : min(img.info.dims[0], img.info.dims[1]);
@@ -458,7 +464,7 @@ void sift(unsigned* out_feat,
 
     std::vector<Param> gauss_pyr = buildGaussPyr<T, convAccT>(init_img, n_octaves, n_layers, init_sigma);
 
-    std::vector<Param> dog_pyr = buildDoGPyr<T>(gauss_pyr, n_octaves, n_layers, suKernel[device]);
+    std::vector<Param> dog_pyr = buildDoGPyr<T>(gauss_pyr, n_octaves, n_layers, kernels[0]);
 
     std::vector<cl::Buffer*> d_x_pyr(n_octaves, NULL);
     std::vector<cl::Buffer*> d_y_pyr(n_octaves, NULL);
@@ -505,7 +511,7 @@ void sift(unsigned* out_feat,
 
         auto deOp = KernelFunctor<Buffer, Buffer, Buffer, Buffer,
                                 Buffer, KParam, unsigned, float,
-                                LocalSpaceArg> (*deKernel[device]);
+                                LocalSpaceArg> (*kernels[1]);
 
         deOp(EnqueueArgs(getQueue(), global, local),
               *d_extrema_x, *d_extrema_y, *d_extrema_layer, *d_count,
@@ -541,7 +547,7 @@ void sift(unsigned* out_feat,
                                 Buffer, Buffer, Buffer,
                                 Buffer, Buffer, Buffer, unsigned,
                                 Buffer, KParam, unsigned, unsigned, unsigned,
-                                float, float, float, float> (*ieKernel[device]);
+                                float, float, float, float> (*kernels[2]);
 
         ieOp(EnqueueArgs(getQueue(), global_interp, local_interp),
               *d_interp_x, *d_interp_y, *d_interp_layer,
@@ -613,7 +619,7 @@ void sift(unsigned* out_feat,
 
         auto rdOp = KernelFunctor<Buffer, Buffer, Buffer, Buffer, Buffer, Buffer,
                                 Buffer, Buffer, Buffer, Buffer, Buffer,
-                                unsigned> (*rdKernel[device]);
+                                unsigned> (*kernels[4]);
 
         rdOp(EnqueueArgs(getQueue(), global_nodup, local_nodup),
               *d_nodup_x, *d_nodup_y, *d_nodup_layer,
@@ -649,7 +655,7 @@ void sift(unsigned* out_feat,
         auto coOp = KernelFunctor<Buffer, Buffer, Buffer, Buffer, Buffer, Buffer, Buffer,
                                 Buffer, Buffer, Buffer, Buffer, Buffer, unsigned,
                                 Buffer, KParam, unsigned, unsigned, int,
-                                LocalSpaceArg> (*coKernel[device]);
+                                LocalSpaceArg> (*kernels[3]);
 
         coOp(EnqueueArgs(getQueue(), global_ori, local_ori),
               *d_oriented_x, *d_oriented_y, *d_oriented_layer,
@@ -694,20 +700,20 @@ void sift(unsigned* out_feat,
             auto cgOp = KernelFunctor<Buffer, unsigned, unsigned,
                                     Buffer, Buffer, Buffer, Buffer, Buffer, Buffer, unsigned,
                                     Buffer, KParam, int, unsigned, unsigned, unsigned, float, int,
-                                    LocalSpaceArg> (*cgKernel[device]);
+                                    LocalSpaceArg> (*kernels[6]);
 
             cgOp(EnqueueArgs(getQueue(), global_desc, local_desc),
-                  *d_desc, desc_len, histsz,
-                  *d_oriented_x, *d_oriented_y, *d_oriented_layer,
-                  *d_oriented_response, *d_oriented_size, *d_oriented_ori, oriented_feat,
-                  *gauss_pyr[o].data, gauss_pyr[o].info, d, rb, ab, hb, scale, n_layers,
-                  cl::Local(desc_len * (histsz+1) * sizeof(float)));
+                 *d_desc, desc_len, histsz,
+                 *d_oriented_x, *d_oriented_y, *d_oriented_layer,
+                 *d_oriented_response, *d_oriented_size, *d_oriented_ori, oriented_feat,
+                 *gauss_pyr[o].data, gauss_pyr[o].info, d, rb, ab, hb, scale, n_layers,
+                 cl::Local(desc_len * (histsz+1) * sizeof(float)));
         }
         else {
             auto cdOp = KernelFunctor<Buffer, unsigned, unsigned,
                                     Buffer, Buffer, Buffer, Buffer, Buffer, Buffer, unsigned,
                                     Buffer, KParam, int, int, float, int,
-                                    LocalSpaceArg> (*cdKernel[device]);
+                                    LocalSpaceArg> (*kernels[5]);
 
             cdOp(EnqueueArgs(getQueue(), global_desc, local_desc),
                   *d_desc, desc_len, histsz,
@@ -816,7 +822,5 @@ void sift(unsigned* out_feat,
     *out_feat = total_feat;
     *out_dlen = desc_len;
 }
-
 } //namespace kernel
-
 } //namespace opencl
diff --git a/src/backend/opencl/kernel/sobel.hpp b/src/backend/opencl/kernel/sobel.hpp
index 2bec2e085f..f77e3ea911 100644
--- a/src/backend/opencl/kernel/sobel.hpp
+++ b/src/backend/opencl/kernel/sobel.hpp
@@ -12,8 +12,7 @@
 #include <program.hpp>
 #include <traits.hpp>
 #include <string>
-#include <mutex>
-#include <map>
+#include <cache.hpp>
 #include <dispatch.hpp>
 #include <Param.hpp>
 #include <debug_opencl.hpp>
@@ -28,60 +27,55 @@ using std::string;
 
 namespace opencl
 {
-
 namespace kernel
 {
-
 static const int THREADS_X = 16;
 static const int THREADS_Y = 16;
 
 template<typename Ti, typename To, unsigned ker_size>
 void sobel(Param dx, Param dy, const Param in)
 {
-    static std::once_flag compileFlags[DeviceManager::MAX_DEVICES];
-    static std::map<int, Program*>  sobProgs;
-    static std::map<int, Kernel*> sobKernels;
+    std::string refName = std::string("sobel3x3_") + std::string(dtype_traits<Ti>::getName()) +
+        std::string(dtype_traits<To>::getName()) + std::to_string(ker_size);
 
     int device = getActiveDeviceId();
-
-    std::call_once( compileFlags[device], [device] () {
-
-            std::ostringstream options;
-            options << " -D Ti=" << dtype_traits<Ti>::getName()
-                    << " -D To=" << dtype_traits<To>::getName()
-                    << " -D KER_SIZE="<< ker_size;
-            if (std::is_same<Ti, double>::value) {
-                options << " -D USE_DOUBLE";
-            }
-            Program prog;
-            buildProgram(prog, sobel_cl, sobel_cl_len, options.str());
-            sobProgs[device]   = new Program(prog);
-            sobKernels[device] = new Kernel(*sobProgs[device], "sobel3x3");
-        });
+    kc_entry_t entry = kernelCache(device, refName);
+
+    if (entry.prog==0 && entry.ker==0) {
+        std::ostringstream options;
+        options << " -D Ti=" << dtype_traits<Ti>::getName()
+                << " -D To=" << dtype_traits<To>::getName()
+                << " -D KER_SIZE="<< ker_size;
+        if (std::is_same<Ti, double>::value)
+            options << " -D USE_DOUBLE";
+
+        const char* ker_strs[] = {sobel_cl};
+        const int   ker_lens[] = {sobel_cl_len};
+        Program prog;
+        buildProgram(prog, 1, ker_strs, ker_lens, options.str());
+        entry.prog = new Program(prog);
+        entry.ker  = new Kernel(*entry.prog, "sobel3x3");
+
+        addKernelToCache(device, refName, entry);
+    }
 
     NDRange local(THREADS_X, THREADS_Y);
 
     int blk_x = divup(in.info.dims[0], THREADS_X);
     int blk_y = divup(in.info.dims[1], THREADS_Y);
 
-    NDRange global(blk_x * in.info.dims[2] * THREADS_X,
-                    blk_y * in.info.dims[3] * THREADS_Y);
+    NDRange global(blk_x * in.info.dims[2] * THREADS_X, blk_y * in.info.dims[3] * THREADS_Y);
 
-    auto sobelOp = KernelFunctor<Buffer, KParam,
-                                Buffer, KParam,
-                                Buffer, KParam,
-                                cl::LocalSpaceArg,
-                                int, int> (*sobKernels[device]);
+    auto sobelOp = KernelFunctor< Buffer, KParam, Buffer, KParam, Buffer, KParam,
+                                  cl::LocalSpaceArg, int, int> (*entry.ker);
 
     size_t loc_size = (THREADS_X+ker_size-1)*(THREADS_Y+ker_size-1)*sizeof(Ti);
 
     sobelOp(EnqueueArgs(getQueue(), global, local),
-                *dx.data, dx.info, *dy.data, dy.info,
-                *in.data, in.info, cl::Local(loc_size), blk_x, blk_y);
+            *dx.data, dx.info, *dy.data, dy.info,
+            *in.data, in.info, cl::Local(loc_size), blk_x, blk_y);
 
     CL_DEBUG_FINISH(getQueue());
 }
-
 }
-
 }
diff --git a/src/backend/opencl/kernel/susan.hpp b/src/backend/opencl/kernel/susan.hpp
index c4d321e01f..bb008bed21 100644
--- a/src/backend/opencl/kernel/susan.hpp
+++ b/src/backend/opencl/kernel/susan.hpp
@@ -14,7 +14,7 @@
 #include <debug_opencl.hpp>
 #include <kernel_headers/susan.hpp>
 #include <memory.hpp>
-#include <map>
+#include <cache.hpp>
 #include "config.hpp"
 
 using cl::Buffer;
@@ -27,10 +27,8 @@ using cl::NDRange;
 
 namespace opencl
 {
-
 namespace kernel
 {
-
 static const unsigned THREADS_PER_BLOCK = 256;
 static const unsigned SUSAN_THREADS_X = 16;
 static const unsigned SUSAN_THREADS_Y = 16;
@@ -41,45 +39,41 @@ void susan(cl::Buffer* out, const cl::Buffer* in,
            const unsigned idim0, const unsigned idim1,
            const float t, const float g, const unsigned edge)
 {
-    static std::once_flag compileFlags[DeviceManager::MAX_DEVICES];
-    static std::map<int, Program*> suProg;
-    static std::map<int, Kernel*>  suKernel;
+    std::string refName = std::string("susan_responses_") +
+        std::string(dtype_traits<T>::getName()) + std::to_string(radius);
 
     int device = getActiveDeviceId();
-
-    std::call_once( compileFlags[device], [device] () {
-
-            const size_t LOCAL_MEM_SIZE = (SUSAN_THREADS_X+2*radius)*(SUSAN_THREADS_Y+2*radius);
-            std::ostringstream options;
-            options << " -D T=" << dtype_traits<T>::getName()
-                    << " -D LOCAL_MEM_SIZE=" << LOCAL_MEM_SIZE
-                    << " -D BLOCK_X="<< SUSAN_THREADS_X
-                    << " -D BLOCK_Y="<< SUSAN_THREADS_Y
-                    << " -D RADIUS="<< radius
-                    << " -D RESPONSE";
-
-            if (std::is_same<T, double>::value ||
-                std::is_same<T, cdouble>::value) {
-                options << " -D USE_DOUBLE";
-            }
-
-            cl::Program prog;
-            buildProgram(prog, susan_cl, susan_cl_len, options.str());
-            suProg[device]   = new Program(prog);
-            suKernel[device] = new Kernel(*suProg[device], "susan_responses");
-        });
-
-    auto susanOp = KernelFunctor<Buffer, Buffer,
-                                unsigned,
-                                unsigned, unsigned,
-                                float, float, unsigned>(*suKernel[device]);
+    kc_entry_t entry = kernelCache(device, refName);
+
+    if (entry.prog==0 && entry.ker==0) {
+        const size_t LOCAL_MEM_SIZE = (SUSAN_THREADS_X+2*radius)*(SUSAN_THREADS_Y+2*radius);
+        std::ostringstream options;
+        options << " -D T=" << dtype_traits<T>::getName()
+                << " -D LOCAL_MEM_SIZE=" << LOCAL_MEM_SIZE
+                << " -D BLOCK_X="<< SUSAN_THREADS_X
+                << " -D BLOCK_Y="<< SUSAN_THREADS_Y
+                << " -D RADIUS="<< radius
+                << " -D RESPONSE";
+        if (std::is_same<T, double>::value || std::is_same<T, cdouble>::value)
+            options << " -D USE_DOUBLE";
+
+        const char* ker_strs[] = {susan_cl};
+        const int   ker_lens[] = {susan_cl_len};
+        Program prog;
+        buildProgram(prog, 1, ker_strs, ker_lens, options.str());
+        entry.prog = new Program(prog);
+        entry.ker  = new Kernel(*entry.prog, "susan_responses");
+
+        addKernelToCache(device, refName, entry);
+    }
+
+    auto susanOp = KernelFunctor< Buffer, Buffer, unsigned, unsigned, unsigned,
+                                  float, float, unsigned >(*entry.ker);
 
     NDRange local(SUSAN_THREADS_X, SUSAN_THREADS_Y);
-    NDRange global(divup(idim0-2*edge, local[0])*local[0],
-                    divup(idim1-2*edge, local[1])*local[1]);
+    NDRange global(divup(idim0-2*edge, local[0])*local[0], divup(idim1-2*edge, local[1])*local[1]);
 
     susanOp(EnqueueArgs(getQueue(), global, local), *out, *in, in_off, idim0, idim1, t, g, edge);
-
 }
 
 template<typename T>
@@ -88,49 +82,45 @@ unsigned nonMaximal(cl::Buffer* x_out, cl::Buffer* y_out, cl::Buffer* resp_out,
                     const unsigned edge, const unsigned max_corners)
 {
     unsigned corners_found = 0;
-    static std::once_flag compileFlags[DeviceManager::MAX_DEVICES];
-    static std::map<int, Program*> nmProg;
-    static std::map<int, Kernel*>  nmKernel;
 
-    int device = getActiveDeviceId();
+    std::string refName = std::string("non_maximal_") + std::string(dtype_traits<T>::getName());
 
-    std::call_once( compileFlags[device], [device] () {
+    int device = getActiveDeviceId();
+    kc_entry_t entry = kernelCache(device, refName);
 
-            std::ostringstream options;
-            options << " -D T=" << dtype_traits<T>::getName()
-                    << " -D NONMAX";
+    if (entry.prog==0 && entry.ker==0) {
+        std::ostringstream options;
+        options << " -D T=" << dtype_traits<T>::getName() << " -D NONMAX";
+        if (std::is_same<T, double>::value || std::is_same<T, cdouble>::value)
+            options << " -D USE_DOUBLE";
 
-            if (std::is_same<T, double>::value ||
-                std::is_same<T, cdouble>::value) {
-                options << " -D USE_DOUBLE";
-            }
+        const char* ker_strs[] = {susan_cl};
+        const int   ker_lens[] = {susan_cl_len};
+        Program prog;
+        buildProgram(prog, 1, ker_strs, ker_lens, options.str());
+        entry.prog = new Program(prog);
+        entry.ker  = new Kernel(*entry.prog, "non_maximal");
 
-            cl::Program prog;
-            buildProgram(prog, susan_cl, susan_cl_len, options.str());
-            nmProg[device]   = new Program(prog);
-            nmKernel[device] = new Kernel(*nmProg[device], "non_maximal");
-        });
+        addKernelToCache(device, refName, entry);
+    }
 
     cl::Buffer *d_corners_found = bufferAlloc(sizeof(unsigned));
     getQueue().enqueueWriteBuffer(*d_corners_found, CL_TRUE, 0, sizeof(unsigned), &corners_found);
 
-    auto nonMaximalOp = KernelFunctor<Buffer, Buffer, Buffer, Buffer,
-                                    unsigned, unsigned, Buffer,
-                                    unsigned, unsigned>(*nmKernel[device]);
+    auto nonMaximalOp = KernelFunctor< Buffer, Buffer, Buffer, Buffer, unsigned, unsigned, Buffer,
+                                       unsigned, unsigned >(*entry.ker);
 
     NDRange local(SUSAN_THREADS_X, SUSAN_THREADS_Y);
-    NDRange global(divup(idim0-2*edge, local[0])*local[0],
-                    divup(idim1-2*edge, local[1])*local[1]);
+    NDRange global(divup(idim0-2*edge, local[0])*local[0], divup(idim1-2*edge, local[1])*local[1]);
 
     nonMaximalOp(EnqueueArgs(getQueue(), global, local),
-                  *x_out, *y_out, *resp_out, *d_corners_found,
-                  idim0, idim1, *resp_in, edge, max_corners);
+                 *x_out, *y_out, *resp_out, *d_corners_found,
+                 idim0, idim1, *resp_in, edge, max_corners);
 
     getQueue().enqueueReadBuffer(*d_corners_found, CL_TRUE, 0, sizeof(unsigned), &corners_found);
     bufferFree(d_corners_found);
+
     return corners_found;
 }
-
 }
-
 }
diff --git a/src/backend/opencl/kernel/swapdblk.hpp b/src/backend/opencl/kernel/swapdblk.hpp
index 0e0f4b16c5..e7c4ab9ed3 100644
--- a/src/backend/opencl/kernel/swapdblk.hpp
+++ b/src/backend/opencl/kernel/swapdblk.hpp
@@ -12,8 +12,7 @@
 #include <program.hpp>
 #include <traits.hpp>
 #include <string>
-#include <mutex>
-#include <map>
+#include <cache.hpp>
 #include <dispatch.hpp>
 #include <Param.hpp>
 #include <debug_opencl.hpp>
@@ -27,41 +26,36 @@ using cl::EnqueueArgs;
 using cl::NDRange;
 using std::string;
 
-
 namespace opencl
 {
-
 namespace kernel
 {
-
 template<typename T>
 void swapdblk(int n, int nb,
               cl_mem dA, size_t dA_offset, int ldda, int inca,
               cl_mem dB, size_t dB_offset, int lddb, int incb)
 {
-
-    static std::once_flag compileFlags[DeviceManager::MAX_DEVICES];
-    static std::map<int, Program*>  swpProgs;
-    static std::map<int, Kernel*> swpKernels;
+    std::string refName = std::string("swapdblk_") + std::string(dtype_traits<T>::getName());
 
     int device = getActiveDeviceId();
+    kc_entry_t entry = kernelCache(device, refName);
 
-    std::call_once(compileFlags[device], [device] () {
+    if (entry.prog==0 && entry.ker==0) {
+        std::ostringstream options;
 
-            std::ostringstream options;
-            options << " -D T=" << dtype_traits<T>::getName();
+        options << " -D T=" << dtype_traits<T>::getName();
+        if (std::is_same<T, double>::value || std::is_same<T, cdouble>::value)
+            options << " -D USE_DOUBLE";
 
-            if (std::is_same<T, double>::value ||
-                std::is_same<T, cdouble>::value) {
-                options << " -D USE_DOUBLE";
-            }
+        const char* ker_strs[] = {swapdblk_cl};
+        const int   ker_lens[] = {swapdblk_cl_len};
+        Program prog;
+        buildProgram(prog, 1, ker_strs, ker_lens, options.str());
+        entry.prog = new Program(prog);
+        entry.ker  = new Kernel(*entry.prog, "swapdblk");
 
-            cl::Program prog;
-            buildProgram(prog, swapdblk_cl, swapdblk_cl_len, options.str());
-            swpProgs[device] = new Program(prog);
-
-            swpKernels[device] = new Kernel(*swpProgs[device], "swapdblk");
-        });
+        addKernelToCache(device, refName, entry);
+    }
 
     int nblocks = n / nb;
 
@@ -94,16 +88,11 @@ void swapdblk(int n, int nb,
     cl::Buffer dAObj(dA, true);
     cl::Buffer dBObj(dB, true);
 
-    auto swapdOp = KernelFunctor<int,
-                               Buffer, unsigned long long, int, int,
-                               Buffer, unsigned long long, int, int>(*swpKernels[device]);
+    auto swapdOp = KernelFunctor<int, Buffer, unsigned long long, int, int,
+                               Buffer, unsigned long long, int, int>(*entry.ker);
 
     swapdOp(EnqueueArgs(getQueue(), global, local),
-            nb,
-            dAObj, dA_offset, ldda, inca,
-            dBObj, dB_offset, lddb, incb);
-
+            nb, dAObj, dA_offset, ldda, inca, dBObj, dB_offset, lddb, incb);
 }
-
 }
 }
diff --git a/src/backend/opencl/kernel/tile.hpp b/src/backend/opencl/kernel/tile.hpp
index 3ad71c21f5..7d15384f86 100644
--- a/src/backend/opencl/kernel/tile.hpp
+++ b/src/backend/opencl/kernel/tile.hpp
@@ -12,8 +12,7 @@
 #include <program.hpp>
 #include <traits.hpp>
 #include <string>
-#include <mutex>
-#include <map>
+#include <cache.hpp>
 #include <dispatch.hpp>
 #include <Param.hpp>
 #include <debug_opencl.hpp>
@@ -28,52 +27,52 @@ using std::string;
 
 namespace opencl
 {
-    namespace kernel
-    {
-        // Kernel Launch Config Values
-        static const int TX = 32;
-        static const int TY = 8;
-        static const int TILEX = 512;
-        static const int TILEY = 32;
+namespace kernel
+{
+// Kernel Launch Config Values
+static const int TX = 32;
+static const int TY = 8;
+static const int TILEX = 512;
+static const int TILEY = 32;
 
-        template<typename T>
-        void tile(Param out, const Param in)
-        {
-            static std::once_flag compileFlags[DeviceManager::MAX_DEVICES];
-            static std::map<int, Program*>   tileProgs;
-            static std::map<int, Kernel *> tileKernels;
+template<typename T>
+void tile(Param out, const Param in)
+{
+    std::string refName = std::string("tile_kernel_") + std::string(dtype_traits<T>::getName());
 
-            int device = getActiveDeviceId();
+    int device = getActiveDeviceId();
+    kc_entry_t entry = kernelCache(device, refName);
 
-            std::call_once( compileFlags[device], [device] () {
-                std::ostringstream options;
-                options << " -D T=" << dtype_traits<T>::getName();
-                if (std::is_same<T, double>::value ||
-                    std::is_same<T, cdouble>::value) {
-                    options << " -D USE_DOUBLE";
-                }
-                Program prog;
-                buildProgram(prog, tile_cl, tile_cl_len, options.str());
-                tileProgs[device] = new Program(prog);
-                tileKernels[device] = new Kernel(*tileProgs[device], "tile_kernel");
-            });
+    if (entry.prog==0 && entry.ker==0) {
+        std::ostringstream options;
+        options << " -D T=" << dtype_traits<T>::getName();
+        if (std::is_same<T, double>::value || std::is_same<T, cdouble>::value)
+            options << " -D USE_DOUBLE";
 
-            auto tileOp = KernelFunctor<Buffer, const Buffer, const KParam, const KParam,
-                                      const int, const int> (*tileKernels[device]);
+        const char* ker_strs[] = {tile_cl};
+        const int   ker_lens[] = {tile_cl_len};
+        Program prog;
+        buildProgram(prog, 1, ker_strs, ker_lens, options.str());
+        entry.prog = new Program(prog);
+        entry.ker  = new Kernel(*entry.prog, "tile_kernel");
 
-            NDRange local(TX, TY, 1);
+        addKernelToCache(device, refName, entry);
+    }
 
-            int blocksPerMatX = divup(out.info.dims[0], TILEX);
-            int blocksPerMatY = divup(out.info.dims[1], TILEY);
-            NDRange global(local[0] * blocksPerMatX * out.info.dims[2],
-                            local[1] * blocksPerMatY * out.info.dims[3],
-                            1);
+    auto tileOp = KernelFunctor< Buffer, const Buffer, const KParam, const KParam,
+                                 const int, const int> (*entry.ker);
 
-            tileOp(EnqueueArgs(getQueue(), global, local),
-                    *out.data, *in.data, out.info, in.info,
-                    blocksPerMatX, blocksPerMatY);
+    NDRange local(TX, TY, 1);
 
-            CL_DEBUG_FINISH(getQueue());
-        }
-    }
+    int blocksPerMatX = divup(out.info.dims[0], TILEX);
+    int blocksPerMatY = divup(out.info.dims[1], TILEY);
+    NDRange global(local[0] * blocksPerMatX * out.info.dims[2],
+                   local[1] * blocksPerMatY * out.info.dims[3], 1);
+
+    tileOp(EnqueueArgs(getQueue(), global, local),
+            *out.data, *in.data, out.info, in.info, blocksPerMatX, blocksPerMatY);
+
+    CL_DEBUG_FINISH(getQueue());
+}
+}
 }
diff --git a/src/backend/opencl/kernel/transform.hpp b/src/backend/opencl/kernel/transform.hpp
index b27445e603..3aa37d97e2 100644
--- a/src/backend/opencl/kernel/transform.hpp
+++ b/src/backend/opencl/kernel/transform.hpp
@@ -13,8 +13,6 @@
 #include <program.hpp>
 #include <traits.hpp>
 #include <string>
-#include <mutex>
-#include <map>
 #include <dispatch.hpp>
 #include <Param.hpp>
 #include <cache.hpp>
@@ -80,8 +78,8 @@ namespace opencl
                         << " -D INVERSE="     << (isInverse ? 1 : 0)
                         << " -D PERSPECTIVE=" << (isPerspective ? 1 : 0)
                         << " -D ZERO="        << toNumStr(scalar<T>(0));
-                options << " -D InterpInTy=" << dtype_traits<T>::getName();
-                options << " -D InterpValTy="  << dtype_traits<vtype_t<T>>::getName();
+                options << " -D InterpInTy="  << dtype_traits<T>::getName();
+                options << " -D InterpValTy=" << dtype_traits<vtype_t<T>>::getName();
                 options << " -D InterpPosTy=" << dtype_traits<wtype_t<BT>>::getName();
 
                 if((af_dtype) dtype_traits<T>::af_type == c32 ||
diff --git a/src/backend/opencl/kernel/transpose.hpp b/src/backend/opencl/kernel/transpose.hpp
index bd4ccfe34f..3fba8c12c8 100644
--- a/src/backend/opencl/kernel/transpose.hpp
+++ b/src/backend/opencl/kernel/transpose.hpp
@@ -12,8 +12,7 @@
 #include <program.hpp>
 #include <traits.hpp>
 #include <string>
-#include <mutex>
-#include <map>
+#include <cache.hpp>
 #include <dispatch.hpp>
 #include <Param.hpp>
 #include <debug_opencl.hpp>
@@ -29,10 +28,8 @@ using std::string;
 
 namespace opencl
 {
-
 namespace kernel
 {
-
 static const int TILE_DIM  = 32;
 static const int THREADS_X = TILE_DIM;
 static const int THREADS_Y = 256 / TILE_DIM;
@@ -40,33 +37,32 @@ static const int THREADS_Y = 256 / TILE_DIM;
 template<typename T, bool conjugate, bool IS32MULTIPLE>
 void transpose(Param out, const Param in)
 {
-    static std::once_flag compileFlags[DeviceManager::MAX_DEVICES];
-    static std::map<int, Program*>  trsProgs;
-    static std::map<int, Kernel*> trsKernels;
+    std::string refName = std::string("transpose_") + std::string(dtype_traits<T>::getName()) +
+        std::to_string(conjugate) + std::to_string(IS32MULTIPLE);
 
     int device = getActiveDeviceId();
+    kc_entry_t entry = kernelCache(device, refName);
 
-    std::call_once(compileFlags[device], [device] () {
-
-            std::ostringstream options;
-            options << " -D TILE_DIM=" << TILE_DIM
-                    << " -D THREADS_Y=" << THREADS_Y
-                    << " -D IS32MULTIPLE=" << IS32MULTIPLE
-                    << " -D DOCONJUGATE=" << (conjugate && af::iscplx<T>())
-                    << " -D T=" << dtype_traits<T>::getName();
+    if (entry.prog==0 && entry.ker==0) {
+        std::ostringstream options;
+        options << " -D TILE_DIM=" << TILE_DIM
+                << " -D THREADS_Y=" << THREADS_Y
+                << " -D IS32MULTIPLE=" << IS32MULTIPLE
+                << " -D DOCONJUGATE=" << (conjugate && af::iscplx<T>())
+                << " -D T=" << dtype_traits<T>::getName();
 
-            if (std::is_same<T, double>::value ||
-                std::is_same<T, cdouble>::value) {
-                options << " -D USE_DOUBLE";
-            }
+        if (std::is_same<T, double>::value || std::is_same<T, cdouble>::value)
+            options << " -D USE_DOUBLE";
 
-            cl::Program prog;
-            buildProgram(prog, transpose_cl, transpose_cl_len, options.str());
-            trsProgs[device] = new Program(prog);
-
-            trsKernels[device] = new Kernel(*trsProgs[device], "transpose");
-        });
+        const char* ker_strs[] = {transpose_cl};
+        const int   ker_lens[] = {transpose_cl_len};
+        Program prog;
+        buildProgram(prog, 1, ker_strs, ker_lens, options.str());
+        entry.prog = new Program(prog);
+        entry.ker  = new Kernel(*entry.prog, "transpose");
 
+        addKernelToCache(device, refName, entry);
+    }
 
     NDRange local(THREADS_X, THREADS_Y);
 
@@ -77,16 +73,13 @@ void transpose(Param out, const Param in)
     NDRange global(blk_x * local[0] * in.info.dims[2],
                     blk_y * local[1] * in.info.dims[3]);
 
-    auto transposeOp = KernelFunctor<Buffer, const KParam,
-                                    const Buffer, const KParam,
-                                    const int, const int> (*trsKernels[device]);
+    auto transposeOp = KernelFunctor< Buffer, const KParam, const Buffer, const KParam,
+                                      const int, const int> (*entry.ker);
 
     transposeOp(EnqueueArgs(getQueue(), global, local),
                 *out.data, out.info, *in.data, in.info, blk_x, blk_y);
 
     CL_DEBUG_FINISH(getQueue());
 }
-
 }
-
 }
diff --git a/src/backend/opencl/kernel/transpose_inplace.hpp b/src/backend/opencl/kernel/transpose_inplace.hpp
index 6784eca86f..b9f7b12a01 100644
--- a/src/backend/opencl/kernel/transpose_inplace.hpp
+++ b/src/backend/opencl/kernel/transpose_inplace.hpp
@@ -12,8 +12,7 @@
 #include <program.hpp>
 #include <traits.hpp>
 #include <string>
-#include <mutex>
-#include <map>
+#include <cache.hpp>
 #include <dispatch.hpp>
 #include <Param.hpp>
 #include <debug_opencl.hpp>
@@ -29,10 +28,8 @@ using std::string;
 
 namespace opencl
 {
-
 namespace kernel
 {
-
 static const int TILE_DIM  = 16;
 static const int THREADS_X = TILE_DIM;
 static const int THREADS_Y = 256 / TILE_DIM;
@@ -40,33 +37,32 @@ static const int THREADS_Y = 256 / TILE_DIM;
 template<typename T, bool conjugate, bool IS32MULTIPLE>
 void transpose_inplace(Param in)
 {
-    static std::once_flag compileFlags[DeviceManager::MAX_DEVICES];
-    static std::map<int, Program*>  transposeProgs;
-    static std::map<int, Kernel*> transposeKernels;
+    std::string refName = std::string("transpose_inplace_") + std::string(dtype_traits<T>::getName()) +
+        std::to_string(conjugate) + std::to_string(IS32MULTIPLE);
 
     int device = getActiveDeviceId();
+    kc_entry_t entry = kernelCache(device, refName);
 
-    std::call_once(compileFlags[device], [device] () {
-
-            std::ostringstream options;
-            options << " -D TILE_DIM=" << TILE_DIM
-                    << " -D THREADS_Y=" << THREADS_Y
-                    << " -D IS32MULTIPLE=" << IS32MULTIPLE
-                    << " -D DOCONJUGATE=" << (conjugate && af::iscplx<T>())
-                    << " -D T=" << dtype_traits<T>::getName();
+    if (entry.prog==0 && entry.ker==0) {
+        std::ostringstream options;
+        options << " -D TILE_DIM=" << TILE_DIM
+                << " -D THREADS_Y=" << THREADS_Y
+                << " -D IS32MULTIPLE=" << IS32MULTIPLE
+                << " -D DOCONJUGATE=" << (conjugate && af::iscplx<T>())
+                << " -D T=" << dtype_traits<T>::getName();
 
-            if (std::is_same<T, double>::value ||
-                std::is_same<T, cdouble>::value) {
-                options << " -D USE_DOUBLE";
-            }
+        if (std::is_same<T, double>::value || std::is_same<T, cdouble>::value)
+            options << " -D USE_DOUBLE";
 
-            cl::Program prog;
-            buildProgram(prog, transpose_inplace_cl, transpose_inplace_cl_len, options.str());
-            transposeProgs[device] = new Program(prog);
-
-            transposeKernels[device] = new Kernel(*transposeProgs[device], "transpose_inplace");
-        });
+        const char* ker_strs[] = {transpose_inplace_cl};
+        const int   ker_lens[] = {transpose_inplace_cl_len};
+        Program prog;
+        buildProgram(prog, 1, ker_strs, ker_lens, options.str());
+        entry.prog = new Program(prog);
+        entry.ker  = new Kernel(*entry.prog, "transpose_inplace");
 
+        addKernelToCache(device, refName, entry);
+    }
 
     NDRange local(THREADS_X, THREADS_Y);
 
@@ -74,17 +70,13 @@ void transpose_inplace(Param in)
     int blk_y = divup(in.info.dims[1], TILE_DIM);
 
     // launch batch * blk_x blocks along x dimension
-    NDRange global(blk_x * local[0] * in.info.dims[2],
-                    blk_y * local[1] * in.info.dims[3]);
+    NDRange global(blk_x * local[0] * in.info.dims[2], blk_y * local[1] * in.info.dims[3]);
 
-    auto transposeOp = KernelFunctor<Buffer, const KParam,
-                                    const int, const int> (*transposeKernels[device]);
+    auto transposeOp = KernelFunctor<Buffer, const KParam, const int, const int> (*entry.ker);
 
     transposeOp(EnqueueArgs(getQueue(), global, local), *in.data, in.info, blk_x, blk_y);
 
     CL_DEBUG_FINISH(getQueue());
 }
-
 }
-
 }
diff --git a/src/backend/opencl/kernel/triangle.hpp b/src/backend/opencl/kernel/triangle.hpp
index acfc4424dd..32925d1cbb 100644
--- a/src/backend/opencl/kernel/triangle.hpp
+++ b/src/backend/opencl/kernel/triangle.hpp
@@ -12,8 +12,7 @@
 #include <program.hpp>
 #include <traits.hpp>
 #include <string>
-#include <mutex>
-#include <map>
+#include <cache.hpp>
 #include <dispatch.hpp>
 #include <Param.hpp>
 #include <debug_opencl.hpp>
@@ -31,10 +30,8 @@ using af::scalar_to_option;
 
 namespace opencl
 {
-
 namespace kernel
 {
-
 // Kernel Launch Config Values
 static const unsigned TX = 32;
 static const unsigned TY = 8;
@@ -44,51 +41,46 @@ static const unsigned TILEY = 32;
 template<typename T, bool is_upper, bool is_unit_diag>
 void triangle(Param out, const Param in)
 {
-    static std::once_flag compileFlags[DeviceManager::MAX_DEVICES];
-    static std::map<int, Program*>  trgProgs;
-    static std::map<int, Kernel*> trgKernels;
+    std::string refName = std::string("triangle_kernel_") + std::string(dtype_traits<T>::getName()) +
+        std::to_string(is_upper) + std::to_string(is_unit_diag);
 
     int device = getActiveDeviceId();
-
-    std::call_once(compileFlags[device], [device] () {
-
-            std::ostringstream options;
-            options << " -D T=" << dtype_traits<T>::getName()
-                    << " -D is_upper=" << is_upper
-                    << " -D is_unit_diag=" << is_unit_diag
-                    << " -D ZERO=(T)(" << scalar_to_option(scalar<T>(0)) << ")"
-                    << " -D ONE=(T)(" << scalar_to_option(scalar<T>(1)) << ")";
-
-            if (std::is_same<T, double>::value ||
-                std::is_same<T, cdouble>::value) {
-                options << " -D USE_DOUBLE";
-            }
-
-            cl::Program prog;
-            buildProgram(prog, triangle_cl, triangle_cl_len, options.str());
-            trgProgs[device] = new Program(prog);
-
-            trgKernels[device] = new Kernel(*trgProgs[device], "triangle_kernel");
-        });
+    kc_entry_t entry = kernelCache(device, refName);
+
+    if (entry.prog==0 && entry.ker==0) {
+        std::ostringstream options;
+        options << " -D T=" << dtype_traits<T>::getName()
+                << " -D is_upper=" << is_upper
+                << " -D is_unit_diag=" << is_unit_diag
+                << " -D ZERO=(T)(" << scalar_to_option(scalar<T>(0)) << ")"
+                << " -D ONE=(T)(" << scalar_to_option(scalar<T>(1)) << ")";
+        if (std::is_same<T, double>::value || std::is_same<T, cdouble>::value)
+            options << " -D USE_DOUBLE";
+
+        const char* ker_strs[] = {triangle_cl};
+        const int   ker_lens[] = {triangle_cl_len};
+        Program prog;
+        buildProgram(prog, 1, ker_strs, ker_lens, options.str());
+        entry.prog = new Program(prog);
+        entry.ker  = new Kernel(*entry.prog, "triangle_kernel");
+
+        addKernelToCache(device, refName, entry);
+    }
 
     NDRange local(TX, TY);
 
     int groups_x = divup(out.info.dims[0], TILEX);
     int groups_y = divup(out.info.dims[1], TILEY);
 
-    NDRange global(groups_x * out.info.dims[2] * local[0],
-                    groups_y * out.info.dims[3] * local[1]);
+    NDRange global(groups_x * out.info.dims[2] * local[0], groups_y * out.info.dims[3] * local[1]);
 
-    auto triangleOp = KernelFunctor<Buffer, KParam,
-                                  const Buffer, KParam,
-                                  const int, const int> (*trgKernels[device]);
+    auto triangleOp = KernelFunctor< Buffer, KParam, const Buffer, KParam,
+                                     const int, const int >(*entry.ker);
 
     triangleOp(EnqueueArgs(getQueue(), global, local),
-                *out.data, out.info, *in.data, in.info, groups_x, groups_y);
+               *out.data, out.info, *in.data, in.info, groups_x, groups_y);
 
     CL_DEBUG_FINISH(getQueue());
 }
-
 }
-
 }
diff --git a/src/backend/opencl/kernel/where.hpp b/src/backend/opencl/kernel/where.hpp
index 26f1bb1ae3..e76943e3c2 100644
--- a/src/backend/opencl/kernel/where.hpp
+++ b/src/backend/opencl/kernel/where.hpp
@@ -9,8 +9,7 @@
 
 #pragma once
 #include <string>
-#include <mutex>
-#include <map>
+#include <cache.hpp>
 #include <kernel_headers/where.hpp>
 #include <program.hpp>
 #include <traits.hpp>
@@ -35,135 +34,122 @@ namespace opencl
 {
 namespace kernel
 {
+template<typename T>
+static void get_out_idx(Buffer *out_data,
+                        Param &otmp, Param &rtmp,
+                        Param &in, uint threads_x,
+                        uint groups_x, uint groups_y)
+{
+    std::string refName = std::string("get_out_idx_kernel_") + std::string(dtype_traits<T>::getName());
+
+    int device = getActiveDeviceId();
+    kc_entry_t entry = kernelCache(device, refName);
+
+    if (entry.prog==0 && entry.ker==0) {
+        ToNumStr<T> toNumStr;
+        std::ostringstream options;
+        options << " -D T=" << dtype_traits<T>::getName()
+                << " -D zero=" << toNumStr(scalar<T>(0))
+                << " -D CPLX=" << af::iscplx<T>();
+        if (std::is_same<T, double>::value || std::is_same<T, cdouble>::value)
+            options << " -D USE_DOUBLE";
+
+        const char* ker_strs[] = {where_cl};
+        const int   ker_lens[] = {where_cl_len};
+        Program prog;
+        buildProgram(prog, 1, ker_strs, ker_lens, options.str());
+        entry.prog = new Program(prog);
+        entry.ker  = new Kernel(*entry.prog, "get_out_idx_kernel");
+
+        addKernelToCache(device, refName, entry);
+    }
 
-    template<typename T>
-    static void get_out_idx(Buffer *out_data,
-                            Param &otmp, Param &rtmp,
-                            Param &in, uint threads_x,
-                            uint groups_x, uint groups_y)
-    {
-        static std::once_flag compileFlags[DeviceManager::MAX_DEVICES];
-        static std::map<int, Program*> whereProgs;
-        static std::map<int, Kernel *> whereKerns;
-
-        int device= getActiveDeviceId();
-
-        std::call_once(compileFlags[device], [device] () {
-
-                ToNumStr<T> toNumStr;
-
-                std::ostringstream options;
-                options << " -D T=" << dtype_traits<T>::getName()
-                        << " -D zero=" << toNumStr(scalar<T>(0))
-                        << " -D CPLX=" << af::iscplx<T>();
-                if (std::is_same<T, double>::value ||
-                    std::is_same<T, cdouble>::value) {
-                    options << " -D USE_DOUBLE";
-                }
-                Program prog;
-                buildProgram(prog, where_cl, where_cl_len, options.str());
-                whereProgs[device] = new Program(prog);
-                whereKerns[device] = new Kernel(*whereProgs[device], "get_out_idx_kernel");
-            });
-
-        NDRange local(threads_x, THREADS_PER_GROUP / threads_x);
-        NDRange global(local[0] * groups_x * in.info.dims[2],
-                       local[1] * groups_y * in.info.dims[3]);
-
-        uint lim = divup(otmp.info.dims[0], (threads_x * groups_x));
-
-        auto whereOp = KernelFunctor<Buffer,
-                                   Buffer, KParam,
-                                   Buffer, KParam,
-                                   Buffer, KParam,
-                                   uint, uint, uint>(*whereKerns[device]);
-
-        whereOp(EnqueueArgs(getQueue(), global, local),
-                *out_data,
-                *otmp.data, otmp.info,
-                *rtmp.data, rtmp.info,
-                *in.data, in.info,
-                groups_x, groups_y, lim);
-
-        CL_DEBUG_FINISH(getQueue());
+    NDRange local(threads_x, THREADS_PER_GROUP / threads_x);
+    NDRange global(local[0] * groups_x * in.info.dims[2], local[1] * groups_y * in.info.dims[3]);
 
-    }
+    uint lim = divup(otmp.info.dims[0], (threads_x * groups_x));
 
-    template<typename T>
-    static void where(Param &out, Param &in)
-    {
-        uint threads_x = nextpow2(std::max(32u, (uint)in.info.dims[0]));
-        threads_x = std::min(threads_x, THREADS_PER_GROUP);
-        uint threads_y = THREADS_PER_GROUP / threads_x;
+    auto whereOp = KernelFunctor< Buffer, Buffer, KParam, Buffer, KParam,
+                                  Buffer, KParam, uint, uint, uint>(*entry.ker);
 
-        uint groups_x = divup(in.info.dims[0], threads_x * REPEAT);
-        uint groups_y = divup(in.info.dims[1], threads_y);
+    whereOp(EnqueueArgs(getQueue(), global, local),
+            *out_data, *otmp.data, otmp.info,
+            *rtmp.data, rtmp.info, *in.data, in.info,
+            groups_x, groups_y, lim);
 
-        Param rtmp;
-        Param otmp;
+    CL_DEBUG_FINISH(getQueue());
+}
 
-        rtmp.info.dims[0] = groups_x;
-        otmp.info.dims[0] = in.info.dims[0];
+template<typename T>
+static void where(Param &out, Param &in)
+{
+    uint threads_x = nextpow2(std::max(32u, (uint)in.info.dims[0]));
+    threads_x = std::min(threads_x, THREADS_PER_GROUP);
+    uint threads_y = THREADS_PER_GROUP / threads_x;
 
-        rtmp.info.strides[0] = 1;
-        otmp.info.strides[0] = 1;
+    uint groups_x = divup(in.info.dims[0], threads_x * REPEAT);
+    uint groups_y = divup(in.info.dims[1], threads_y);
 
-        rtmp.info.offset = 0;
-        otmp.info.offset = 0;
+    Param rtmp;
+    Param otmp;
 
-        for (int k = 1; k < 4; k++) {
-            rtmp.info.dims[k] = in.info.dims[k];
-            rtmp.info.strides[k] = rtmp.info.strides[k - 1] * rtmp.info.dims[k - 1];
+    rtmp.info.dims[0] = groups_x;
+    otmp.info.dims[0] = in.info.dims[0];
 
-            otmp.info.dims[k] = in.info.dims[k];
-            otmp.info.strides[k] = otmp.info.strides[k - 1] * otmp.info.dims[k - 1];
-        }
+    rtmp.info.strides[0] = 1;
+    otmp.info.strides[0] = 1;
 
-        int rtmp_elements = rtmp.info.strides[3] * rtmp.info.dims[3];
-        rtmp.data = bufferAlloc(rtmp_elements * sizeof(uint));
+    rtmp.info.offset = 0;
+    otmp.info.offset = 0;
 
-        int otmp_elements = otmp.info.strides[3] * otmp.info.dims[3];
-        otmp.data = bufferAlloc(otmp_elements * sizeof(uint));
+    for (int k = 1; k < 4; k++) {
+        rtmp.info.dims[k] = in.info.dims[k];
+        rtmp.info.strides[k] = rtmp.info.strides[k - 1] * rtmp.info.dims[k - 1];
 
-        scan_first_launcher<T, uint, af_notzero_t>(otmp, rtmp, in,
-                                                    false,
-                                                    groups_x, groups_y,
-                                                    threads_x);
+        otmp.info.dims[k] = in.info.dims[k];
+        otmp.info.strides[k] = otmp.info.strides[k - 1] * otmp.info.dims[k - 1];
+    }
 
-        // Linearize the dimensions and perform scan
-        Param ltmp = rtmp;
-        ltmp.info.offset = 0;
-        ltmp.info.dims[0] = rtmp_elements;
-        for (int k = 1; k < 4; k++) {
-            ltmp.info.dims[k] = 1;
-            ltmp.info.strides[k] = rtmp_elements;
-        }
+    int rtmp_elements = rtmp.info.strides[3] * rtmp.info.dims[3];
+    rtmp.data = bufferAlloc(rtmp_elements * sizeof(uint));
 
-        scan_first<uint, uint, af_add_t>(ltmp, ltmp);
+    int otmp_elements = otmp.info.strides[3] * otmp.info.dims[3];
+    otmp.data = bufferAlloc(otmp_elements * sizeof(uint));
 
-        // Get output size and allocate output
-        uint total;
-        getQueue().enqueueReadBuffer(*rtmp.data, CL_TRUE,
-                                      sizeof(uint) * (rtmp_elements - 1),
-                                      sizeof(uint),
-                                      &total);
+    scan_first_launcher<T, uint, af_notzero_t>(otmp, rtmp, in, false, groups_x, groups_y, threads_x);
 
+    // Linearize the dimensions and perform scan
+    Param ltmp = rtmp;
+    ltmp.info.offset = 0;
+    ltmp.info.dims[0] = rtmp_elements;
+    for (int k = 1; k < 4; k++) {
+        ltmp.info.dims[k] = 1;
+        ltmp.info.strides[k] = rtmp_elements;
+    }
 
-        out.data = bufferAlloc(total * sizeof(uint));
+    scan_first<uint, uint, af_add_t>(ltmp, ltmp);
 
-        out.info.dims[0] = total;
-        out.info.strides[0] = 1;
-        for (int k = 1; k < 4; k++) {
-            out.info.dims[k] = 1;
-            out.info.strides[k] = total;
-        }
+    // Get output size and allocate output
+    uint total;
+    getQueue().enqueueReadBuffer(*rtmp.data, CL_TRUE,
+                                  sizeof(uint) * (rtmp_elements - 1),
+                                  sizeof(uint),
+                                  &total);
 
-        if (total > 0) {
-            get_out_idx<T>(out.data, otmp, rtmp, in, threads_x, groups_x, groups_y);
-        }
+    out.data = bufferAlloc(total * sizeof(uint));
 
-        bufferFree(rtmp.data);
-        bufferFree(otmp.data);
+    out.info.dims[0] = total;
+    out.info.strides[0] = 1;
+    for (int k = 1; k < 4; k++) {
+        out.info.dims[k] = 1;
+        out.info.strides[k] = total;
     }
+
+    if (total > 0)
+        get_out_idx<T>(out.data, otmp, rtmp, in, threads_x, groups_x, groups_y);
+
+    bufferFree(rtmp.data);
+    bufferFree(otmp.data);
+}
 }
 }

From f3f083e0e0eea6f04726de5c6bc52e40b7e0d0cc Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Thu, 13 Apr 2017 13:39:31 +0530
Subject: [PATCH 63/82] Move clBlas library initilisation to DeviceManager
 constructor

* Re-enabled FFT, BLAS multi-threaded tests for OpenCL backend.
* clblasSetup() call is not thread safe. Having it initialsed from the
  DeviceManager makes more sense instead of using call_once mechanism.
* Split FFT tests into R2C and C2C categories.
---
 src/backend/opencl/blas.cpp     |   5 --
 src/backend/opencl/blas.hpp     |   8 ---
 src/backend/opencl/cholesky.cpp |   2 -
 src/backend/opencl/lu.cpp       |   1 -
 src/backend/opencl/platform.cpp |   6 ++
 src/backend/opencl/qr.cpp       |   2 -
 src/backend/opencl/solve.cpp    |   2 -
 src/backend/opencl/svd.cpp      |   1 -
 test/threading.cpp              | 109 ++++++++++++--------------------
 9 files changed, 48 insertions(+), 88 deletions(-)

diff --git a/src/backend/opencl/blas.cpp b/src/backend/opencl/blas.cpp
index 420ee1f332..986e2c226d 100644
--- a/src/backend/opencl/blas.cpp
+++ b/src/backend/opencl/blas.cpp
@@ -13,7 +13,6 @@
 #include <string>
 #include <functional>
 #include <stdexcept>
-#include <mutex>
 #include <err_opencl.hpp>
 #include <err_clblas.hpp>
 #include <math.hpp>
@@ -31,8 +30,6 @@ namespace opencl
 
 using std::is_floating_point;
 using std::enable_if;
-using std::once_flag;
-using std::call_once;
 using std::runtime_error;
 using std::to_string;
 
@@ -124,8 +121,6 @@ Array<T> matmul(const Array<T> &lhs, const Array<T> &rhs,
         return cpu::matmul(lhs, rhs, optLhs, optRhs);
     }
 #endif
-
-    initBlas();
     clblasTranspose lOpts = toClblasTranspose(optLhs);
     clblasTranspose rOpts = toClblasTranspose(optRhs);
 
diff --git a/src/backend/opencl/blas.hpp b/src/backend/opencl/blas.hpp
index f6676abeff..e9408c4b77 100644
--- a/src/backend/opencl/blas.hpp
+++ b/src/backend/opencl/blas.hpp
@@ -10,21 +10,13 @@
 #pragma once
 #include <Array.hpp>
 #include <clBLAS.h>
-#include <mutex>
 
 namespace opencl
 {
-
 template<typename T>
 Array<T> matmul(const Array<T> &lhs, const Array<T> &rhs,
                 af_mat_prop optLhs, af_mat_prop optRhs);
 template<typename T>
 Array<T> dot(const Array<T> &lhs, const Array<T> &rhs,
              af_mat_prop optLhs, af_mat_prop optRhs);
-
-STATIC_ void
-initBlas() {
-    static std::once_flag clblasSetupFlag;
-    call_once(clblasSetupFlag, clblasSetup);
-}
 }
diff --git a/src/backend/opencl/cholesky.cpp b/src/backend/opencl/cholesky.cpp
index f949b85a01..47df889539 100644
--- a/src/backend/opencl/cholesky.cpp
+++ b/src/backend/opencl/cholesky.cpp
@@ -28,8 +28,6 @@ int cholesky_inplace(Array<T> &in, const bool is_upper)
         return cpu::cholesky_inplace(in, is_upper);
     }
 
-    initBlas();
-
     dim4 iDims = in.dims();
     int N = iDims[0];
 
diff --git a/src/backend/opencl/lu.cpp b/src/backend/opencl/lu.cpp
index 70b6d97f41..aa8cfeef3d 100644
--- a/src/backend/opencl/lu.cpp
+++ b/src/backend/opencl/lu.cpp
@@ -71,7 +71,6 @@ Array<int> lu_inplace(Array<T> &in, const bool convert_pivot)
         return cpu::lu_inplace(in, convert_pivot);
     }
 
-    initBlas();
     dim4 iDims = in.dims();
     int M = iDims[0];
     int N = iDims[1];
diff --git a/src/backend/opencl/platform.cpp b/src/backend/opencl/platform.cpp
index 3ea48f4ac6..24bdefc2ea 100644
--- a/src/backend/opencl/platform.cpp
+++ b/src/backend/opencl/platform.cpp
@@ -27,6 +27,7 @@
 #include <defines.hpp>
 #include <errorcodes.hpp>
 #include <err_opencl.hpp>
+#include <err_clblas.hpp>
 #include <host_memory.hpp>
 #include <common/InteropManager.hpp>
 #include <platform.hpp>
@@ -903,6 +904,11 @@ DeviceManager::DeviceManager()
     //Initialize FFT setup data structure
     CLFFT_CHECK(clfftInitSetupData(&mFFTSetup));
     CLFFT_CHECK(clfftSetup(&mFFTSetup));
+
+    //Initialize clBlas library, clblasSetup is not thread safe
+    //Since DeviceManager class is singleton, it is okay to call
+    //without any synchronization mechanisms
+    CLBLAS_CHECK(clblasSetup());
 }
 
 #if defined(WITH_GRAPHICS)
diff --git a/src/backend/opencl/qr.cpp b/src/backend/opencl/qr.cpp
index cb730122db..2f3df19dee 100644
--- a/src/backend/opencl/qr.cpp
+++ b/src/backend/opencl/qr.cpp
@@ -32,7 +32,6 @@ void qr(Array<T> &q, Array<T> &r, Array<T> &t, const Array<T> &orig)
         return cpu::qr(q, r, t, orig);
     }
 
-    initBlas();
     dim4 iDims = orig.dims();
     int M = iDims[0];
     int N = iDims[1];
@@ -86,7 +85,6 @@ Array<T> qr_inplace(Array<T> &in)
         return cpu::qr_inplace(in);
     }
 
-    initBlas();
     dim4 iDims = in.dims();
     int M = iDims[0];
     int N = iDims[1];
diff --git a/src/backend/opencl/solve.cpp b/src/backend/opencl/solve.cpp
index 61de3dc692..068321f36c 100644
--- a/src/backend/opencl/solve.cpp
+++ b/src/backend/opencl/solve.cpp
@@ -304,8 +304,6 @@ Array<T> solve(const Array<T> &a, const Array<T> &b, const af_mat_prop options)
         return cpu::solve(a, b, options);
     }
 
-    initBlas();
-
     if (options & AF_MAT_UPPER ||
         options & AF_MAT_LOWER) {
         return triangleSolve<T>(a, b, options);
diff --git a/src/backend/opencl/svd.cpp b/src/backend/opencl/svd.cpp
index 61da27bdcd..bc0043ee10 100644
--- a/src/backend/opencl/svd.cpp
+++ b/src/backend/opencl/svd.cpp
@@ -202,7 +202,6 @@ void svdInPlace(Array<Tr> &s, Array<T> &u, Array<T> &vt, Array<T> &in)
         return cpu::svdInPlace(s, u, vt, in);
     }
 
-    initBlas();
     svd<T, Tr>(u, s, vt, in, true);
 }
 
diff --git a/test/threading.cpp b/test/threading.cpp
index d5df9cf0c1..f4c415e64d 100644
--- a/test/threading.cpp
+++ b/test/threading.cpp
@@ -42,8 +42,6 @@ int nextTargetDeviceId()
 void morphTest(const array input, const array mask, const bool isDilation,
                const array gold, int targetDevice)
 {
-    auto start = std::chrono::high_resolution_clock::now();
-
     af::setDevice(targetDevice);
 
     vector<float> goldData(gold.elements());
@@ -59,16 +57,6 @@ void morphTest(const array input, const array mask, const bool isDilation,
     out.host((void*)outData.data());
 
     ASSERT_EQ(true, compareArraysRMSD(gold.elements(), goldData.data(), outData.data(), 0.018f));
-
-    auto end = std::chrono::high_resolution_clock::now();
-
-    std::chrono::duration<double> diff = end - start;
-
-    std::cout << "Thread(" << std::this_thread::get_id()
-              << "): time taken for "
-              << ITERATION_COUNT
-              <<" is "
-              << diff.count() << " s\n";
 }
 
 TEST(Threading, SetPerThreadActiveDevice)
@@ -90,8 +78,6 @@ TEST(Threading, SetPerThreadActiveDevice)
     vector<std::thread> tests;
     unsigned totalTestCount = 0;
 
-    auto start = std::chrono::high_resolution_clock::now();
-
     for(size_t pos = 0; pos<files.size(); ++pos)
     {
         const bool isDilation = isDilationFlags[pos];
@@ -126,32 +112,13 @@ TEST(Threading, SetPerThreadActiveDevice)
             //Push the new test as a new thread of execution
             tests.emplace_back(morphTest, input, mask, isDilation, gold, trgtDeviceId);
 
-            std::cout<<"morph test launched with the following params on device ("
-                     <<trgtDeviceId<<"):"<<std::endl;
-            std::cout<<"\t Input image dims: "<<input.dims()<<std::endl;
-            std::cout<<"\t Mask dims: "<<mask.dims()<<std::endl;
-            std::cout<<"\t IsDilation : "<< (isDilation ? "True" : "False") <<std::endl;
-
             totalTestCount++;
         }
     }
-    std::cout<< std::endl << "Waiting for results ..." << std::endl << std::endl;
 
     for (size_t testId=0; testId<tests.size(); ++testId)
-    {
-        if (tests[testId].joinable()) {
-            std::cout<<"Attempting join for test ..."<<testId<<std::endl;
+        if (tests[testId].joinable())
             tests[testId].join();
-            std::cout<<"test "<< testId <<" completed." << std::endl;
-        }
-        std::cout<<std::endl;
-    }
-
-    auto end = std::chrono::high_resolution_clock::now();
-
-    std::chrono::duration<double> diff = end - start;
-
-    std::cout << "Total time taken for test : " << diff.count() << " s\n";
 }
 
 enum ArithOp
@@ -341,7 +308,6 @@ TEST(Threading, MemoryManagement_JIT_Node)
     ASSERT_EQ(  lock_bytes,      0u);
 }
 
-#if !defined(AF_OPENCL)
 template<typename inType, typename outType, bool isInverse>
 void fftTest(int targetDevice, string pTestFile, dim_t pad0=0, dim_t pad1=0, dim_t pad2=0)
 {
@@ -419,12 +385,10 @@ void fftTest(int targetDevice, string pTestFile, dim_t pad0=0, dim_t pad1=0, dim
         tests.emplace_back(fftTest<in_t, out_t, is_inverse>, targetDevice, file, p0, p1, 0);\
     }
 
-/// OpenCL backend tests seem to be failing even when
-/// each thead has it's own plan cache(thread_local).
-/// The issue seems to present itself randomly in the form of crashes,
-/// garbage values.
-TEST(Threading, FFT)
+TEST(Threading, FFT_R2C)
 {
+    cleanSlate(); // Clean up everything done so far
+
     vector<std::thread> tests;
 
     int numDevices = 1;
@@ -438,14 +402,6 @@ TEST(Threading, FFT)
     INSTANTIATE_TEST(fft3,  R2C_Float, false,  float,  cfloat, string(TEST_DIR"/signal/fft3_r2c.test"));
     INSTANTIATE_TEST(fft3, R2C_Double, false, double, cdouble, string(TEST_DIR"/signal/fft3_r2c.test"));
 
-    // complex to complex transforms
-    INSTANTIATE_TEST(fft ,  C2C_Float, false,  cfloat,  cfloat, string(TEST_DIR"/signal/fft_c2c.test") );
-    INSTANTIATE_TEST(fft , C2C_Double, false, cdouble, cdouble, string(TEST_DIR"/signal/fft_c2c.test") );
-    INSTANTIATE_TEST(fft2,  C2C_Float, false,  cfloat,  cfloat, string(TEST_DIR"/signal/fft2_c2c.test"));
-    INSTANTIATE_TEST(fft2, C2C_Double, false, cdouble, cdouble, string(TEST_DIR"/signal/fft2_c2c.test"));
-    INSTANTIATE_TEST(fft3,  C2C_Float, false,  cfloat,  cfloat, string(TEST_DIR"/signal/fft3_c2c.test"));
-    INSTANTIATE_TEST(fft3, C2C_Double, false, cdouble, cdouble, string(TEST_DIR"/signal/fft3_c2c.test"));
-
     // Factors 7, 11, 13
     INSTANTIATE_TEST(fft , R2C_Float_7_11_13 , false, float  , cfloat , string(TEST_DIR"/signal/fft_r2c_7_11_13.test") );
     INSTANTIATE_TEST(fft , R2C_Double_7_11_13, false, double , cdouble, string(TEST_DIR"/signal/fft_r2c_7_11_13.test") );
@@ -454,6 +410,32 @@ TEST(Threading, FFT)
     INSTANTIATE_TEST(fft3, R2C_Float_7_11_13 , false, float  , cfloat , string(TEST_DIR"/signal/fft3_r2c_7_11_13.test") );
     INSTANTIATE_TEST(fft3, R2C_Double_7_11_13, false, double , cdouble, string(TEST_DIR"/signal/fft3_r2c_7_11_13.test") );
 
+    // transforms on padded and truncated arrays
+    INSTANTIATE_TEST_TP(fft2,  R2C_Float_Trunc, false,  float,  cfloat, string(TEST_DIR"/signal/fft2_r2c_trunc.test"), 16, 16);
+    INSTANTIATE_TEST_TP(fft2, R2C_Double_Trunc, false, double, cdouble, string(TEST_DIR"/signal/fft2_r2c_trunc.test"), 16, 16);
+
+    for (size_t testId=0; testId<tests.size(); ++testId)
+        if (tests[testId].joinable())
+            tests[testId].join();
+}
+
+TEST(Threading, FFT_C2C)
+{
+    cleanSlate(); // Clean up everything done so far
+
+    vector<std::thread> tests;
+
+    int numDevices = 1;
+    ASSERT_EQ(AF_SUCCESS, af_get_device_count(&numDevices));
+
+    // complex to complex transforms
+    INSTANTIATE_TEST(fft ,  C2C_Float, false,  cfloat,  cfloat, string(TEST_DIR"/signal/fft_c2c.test") );
+    INSTANTIATE_TEST(fft , C2C_Double, false, cdouble, cdouble, string(TEST_DIR"/signal/fft_c2c.test") );
+    INSTANTIATE_TEST(fft2,  C2C_Float, false,  cfloat,  cfloat, string(TEST_DIR"/signal/fft2_c2c.test"));
+    INSTANTIATE_TEST(fft2, C2C_Double, false, cdouble, cdouble, string(TEST_DIR"/signal/fft2_c2c.test"));
+    INSTANTIATE_TEST(fft3,  C2C_Float, false,  cfloat,  cfloat, string(TEST_DIR"/signal/fft3_c2c.test"));
+    INSTANTIATE_TEST(fft3, C2C_Double, false, cdouble, cdouble, string(TEST_DIR"/signal/fft3_c2c.test"));
+
     INSTANTIATE_TEST(fft , C2C_Float_7_11_13 , false, cfloat  , cfloat , string(TEST_DIR"/signal/fft_c2c_7_11_13.test") );
     INSTANTIATE_TEST(fft , C2C_Double_7_11_13, false, cdouble , cdouble, string(TEST_DIR"/signal/fft_c2c_7_11_13.test") );
     INSTANTIATE_TEST(fft2, C2C_Float_7_11_13 , false, cfloat  , cfloat , string(TEST_DIR"/signal/fft2_c2c_7_11_13.test") );
@@ -462,9 +444,6 @@ TEST(Threading, FFT)
     INSTANTIATE_TEST(fft3, C2C_Double_7_11_13, false, cdouble , cdouble, string(TEST_DIR"/signal/fft3_c2c_7_11_13.test") );
 
     // transforms on padded and truncated arrays
-    INSTANTIATE_TEST_TP(fft2,  R2C_Float_Trunc, false,  float,  cfloat, string(TEST_DIR"/signal/fft2_r2c_trunc.test"), 16, 16);
-    INSTANTIATE_TEST_TP(fft2, R2C_Double_Trunc, false, double, cdouble, string(TEST_DIR"/signal/fft2_r2c_trunc.test"), 16, 16);
-
     INSTANTIATE_TEST_TP(fft2,  C2C_Float_Pad, false,  cfloat,  cfloat, string(TEST_DIR"/signal/fft2_c2c_pad.test"), 16, 16);
     INSTANTIATE_TEST_TP(fft2, C2C_Double_Pad, false, cdouble, cdouble, string(TEST_DIR"/signal/fft2_c2c_pad.test"), 16, 16);
 
@@ -478,11 +457,8 @@ TEST(Threading, FFT)
     INSTANTIATE_TEST(ifft3, C2C_Double, true, cdouble, cdouble, string(TEST_DIR"/signal/ifft3_c2c.test"));
 
     for (size_t testId=0; testId<tests.size(); ++testId)
-    {
-        if (tests[testId].joinable()) {
+        if (tests[testId].joinable())
             tests[testId].join();
-        }
-    }
 }
 
 template<typename T, bool isBVector>
@@ -561,6 +537,8 @@ void cppMatMulCheck(int targetDevice, string TestFile)
 
 TEST(Threading, BLAS)
 {
+    cleanSlate(); // Clean up everything done so far
+
     vector<std::thread> tests;
 
     int numDevices = 1;
@@ -572,13 +550,12 @@ TEST(Threading, BLAS)
     TEST_FOR_TYPE(af::cdouble);
 
     for (size_t testId=0; testId<tests.size(); ++testId)
-    {
-        if (tests[testId].joinable()) {
+        if (tests[testId].joinable())
             tests[testId].join();
-        }
-    }
 }
 
+#if !defined(AF_OPENCL)
+
 #define SOLVE_LU_TESTS(T, eps)                                                                          \
     tests.emplace_back(solveLUTester<T>, 1000, 100, eps, nextTargetDeviceId()%numDevices);              \
     tests.emplace_back(solveLUTester<T>, 2048, 512, eps, nextTargetDeviceId()%numDevices);              \
@@ -602,6 +579,8 @@ TEST(Threading, BLAS)
 // we are not running out of memory.
 TEST(Threading, SolveDense)
 {
+    cleanSlate(); // Clean up everything done so far
+
     vector<std::thread> tests;
 
     int numDevices = 1;
@@ -613,11 +592,8 @@ TEST(Threading, SolveDense)
     SOLVE_LU_TESTS(cdouble, 1E-5);
 
     for (size_t testId=0; testId<tests.size(); ++testId)
-    {
-        if (tests[testId].joinable()) {
+        if (tests[testId].joinable())
             tests[testId].join();
-        }
-    }
 }
 
 #undef SOLVE_LU_TESTS
@@ -638,6 +614,8 @@ TEST(Threading, SolveDense)
 // we are not running out of memory.
 TEST(Threading, Sparse)
 {
+    cleanSlate(); // Clean up everything done so far
+
     vector<std::thread> tests;
 
     int numDevices = 1;
@@ -649,11 +627,8 @@ TEST(Threading, Sparse)
     SPARSE_TESTS(cdouble, 1E-5);
 
     for (size_t testId=0; testId<tests.size(); ++testId)
-    {
-        if (tests[testId].joinable()) {
+        if (tests[testId].joinable())
             tests[testId].join();
-        }
-    }
 }
 
 #endif

From b1627dd38e5682508919c39ec8962321aa6388d0 Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Sun, 16 Apr 2017 12:05:37 +0530
Subject: [PATCH 64/82] Add FFT multi-threaded test for all cases

---
 test/threading.cpp | 62 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 62 insertions(+)

diff --git a/test/threading.cpp b/test/threading.cpp
index f4c415e64d..181b1270b3 100644
--- a/test/threading.cpp
+++ b/test/threading.cpp
@@ -461,6 +461,68 @@ TEST(Threading, FFT_C2C)
             tests[testId].join();
 }
 
+TEST(Threading, FFT_ALL)
+{
+    cleanSlate(); // Clean up everything done so far
+
+    vector<std::thread> tests;
+
+    int numDevices = 1;
+    ASSERT_EQ(AF_SUCCESS, af_get_device_count(&numDevices));
+
+    // Real to complex transforms
+    INSTANTIATE_TEST(fft ,  R2C_Float, false,  float,  cfloat, string(TEST_DIR"/signal/fft_r2c.test") );
+    INSTANTIATE_TEST(fft , R2C_Double, false, double, cdouble, string(TEST_DIR"/signal/fft_r2c.test") );
+    INSTANTIATE_TEST(fft2,  R2C_Float, false,  float,  cfloat, string(TEST_DIR"/signal/fft2_r2c.test"));
+    INSTANTIATE_TEST(fft2, R2C_Double, false, double, cdouble, string(TEST_DIR"/signal/fft2_r2c.test"));
+    INSTANTIATE_TEST(fft3,  R2C_Float, false,  float,  cfloat, string(TEST_DIR"/signal/fft3_r2c.test"));
+    INSTANTIATE_TEST(fft3, R2C_Double, false, double, cdouble, string(TEST_DIR"/signal/fft3_r2c.test"));
+
+    // Factors 7, 11, 13
+    INSTANTIATE_TEST(fft , R2C_Float_7_11_13 , false, float  , cfloat , string(TEST_DIR"/signal/fft_r2c_7_11_13.test") );
+    INSTANTIATE_TEST(fft , R2C_Double_7_11_13, false, double , cdouble, string(TEST_DIR"/signal/fft_r2c_7_11_13.test") );
+    INSTANTIATE_TEST(fft2, R2C_Float_7_11_13 , false, float  , cfloat , string(TEST_DIR"/signal/fft2_r2c_7_11_13.test") );
+    INSTANTIATE_TEST(fft2, R2C_Double_7_11_13, false, double , cdouble, string(TEST_DIR"/signal/fft2_r2c_7_11_13.test") );
+    INSTANTIATE_TEST(fft3, R2C_Float_7_11_13 , false, float  , cfloat , string(TEST_DIR"/signal/fft3_r2c_7_11_13.test") );
+    INSTANTIATE_TEST(fft3, R2C_Double_7_11_13, false, double , cdouble, string(TEST_DIR"/signal/fft3_r2c_7_11_13.test") );
+
+    // transforms on padded and truncated arrays
+    INSTANTIATE_TEST_TP(fft2,  R2C_Float_Trunc, false,  float,  cfloat, string(TEST_DIR"/signal/fft2_r2c_trunc.test"), 16, 16);
+    INSTANTIATE_TEST_TP(fft2, R2C_Double_Trunc, false, double, cdouble, string(TEST_DIR"/signal/fft2_r2c_trunc.test"), 16, 16);
+
+    // complex to complex transforms
+    INSTANTIATE_TEST(fft ,  C2C_Float, false,  cfloat,  cfloat, string(TEST_DIR"/signal/fft_c2c.test") );
+    INSTANTIATE_TEST(fft , C2C_Double, false, cdouble, cdouble, string(TEST_DIR"/signal/fft_c2c.test") );
+    INSTANTIATE_TEST(fft2,  C2C_Float, false,  cfloat,  cfloat, string(TEST_DIR"/signal/fft2_c2c.test"));
+    INSTANTIATE_TEST(fft2, C2C_Double, false, cdouble, cdouble, string(TEST_DIR"/signal/fft2_c2c.test"));
+    INSTANTIATE_TEST(fft3,  C2C_Float, false,  cfloat,  cfloat, string(TEST_DIR"/signal/fft3_c2c.test"));
+    INSTANTIATE_TEST(fft3, C2C_Double, false, cdouble, cdouble, string(TEST_DIR"/signal/fft3_c2c.test"));
+
+    INSTANTIATE_TEST(fft , C2C_Float_7_11_13 , false, cfloat  , cfloat , string(TEST_DIR"/signal/fft_c2c_7_11_13.test") );
+    INSTANTIATE_TEST(fft , C2C_Double_7_11_13, false, cdouble , cdouble, string(TEST_DIR"/signal/fft_c2c_7_11_13.test") );
+    INSTANTIATE_TEST(fft2, C2C_Float_7_11_13 , false, cfloat  , cfloat , string(TEST_DIR"/signal/fft2_c2c_7_11_13.test") );
+    INSTANTIATE_TEST(fft2, C2C_Double_7_11_13, false, cdouble , cdouble, string(TEST_DIR"/signal/fft2_c2c_7_11_13.test") );
+    INSTANTIATE_TEST(fft3, C2C_Float_7_11_13 , false, cfloat  , cfloat , string(TEST_DIR"/signal/fft3_c2c_7_11_13.test") );
+    INSTANTIATE_TEST(fft3, C2C_Double_7_11_13, false, cdouble , cdouble, string(TEST_DIR"/signal/fft3_c2c_7_11_13.test") );
+
+    // transforms on padded and truncated arrays
+    INSTANTIATE_TEST_TP(fft2,  C2C_Float_Pad, false,  cfloat,  cfloat, string(TEST_DIR"/signal/fft2_c2c_pad.test"), 16, 16);
+    INSTANTIATE_TEST_TP(fft2, C2C_Double_Pad, false, cdouble, cdouble, string(TEST_DIR"/signal/fft2_c2c_pad.test"), 16, 16);
+
+    // inverse transforms
+    // complex to complex transforms
+    INSTANTIATE_TEST(ifft ,  C2C_Float, true,  cfloat,  cfloat, string(TEST_DIR"/signal/ifft_c2c.test") );
+    INSTANTIATE_TEST(ifft , C2C_Double, true, cdouble, cdouble, string(TEST_DIR"/signal/ifft_c2c.test") );
+    INSTANTIATE_TEST(ifft2,  C2C_Float, true,  cfloat,  cfloat, string(TEST_DIR"/signal/ifft2_c2c.test"));
+    INSTANTIATE_TEST(ifft2, C2C_Double, true, cdouble, cdouble, string(TEST_DIR"/signal/ifft2_c2c.test"));
+    INSTANTIATE_TEST(ifft3,  C2C_Float, true,  cfloat,  cfloat, string(TEST_DIR"/signal/ifft3_c2c.test"));
+    INSTANTIATE_TEST(ifft3, C2C_Double, true, cdouble, cdouble, string(TEST_DIR"/signal/ifft3_c2c.test"));
+
+    for (size_t testId=0; testId<tests.size(); ++testId)
+        if (tests[testId].joinable())
+            tests[testId].join();
+}
+
 template<typename T, bool isBVector>
 void cppMatMulCheck(int targetDevice, string TestFile)
 {

From 41a3e056ccf3e1a747c75b65d89d3fee6d6c6e2e Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Sun, 16 Apr 2017 16:34:36 +0530
Subject: [PATCH 65/82] Add global mutex to serialize clblas calls

---
 src/backend/opencl/err_clblas.hpp | 7 ++++++-
 src/backend/opencl/platform.cpp   | 6 ++++++
 test/threading.cpp                | 4 ----
 3 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/src/backend/opencl/err_clblas.hpp b/src/backend/opencl/err_clblas.hpp
index 4d28f94960..768ead7856 100644
--- a/src/backend/opencl/err_clblas.hpp
+++ b/src/backend/opencl/err_clblas.hpp
@@ -11,6 +11,7 @@
 #include <stdio.h>
 #include <err_opencl.hpp>
 #include <clBLAS.h>
+#include <mutex>
 
 static const char * _clblasGetResultString(clblasStatus st)
 {
@@ -51,9 +52,12 @@ static const char * _clblasGetResultString(clblasStatus st)
     return "Unknown error";
 }
 
+static std::recursive_mutex gCLBlasMutex;
+
 #define CLBLAS_CHECK(fn) do {                   \
+        gCLBlasMutex.lock();                    \
         clblasStatus _clblas_st = fn;           \
-        if (_clblas_st != clblasSuccess) {     \
+        if (_clblas_st != clblasSuccess) {      \
             char clblas_st_msg[1024];           \
             snprintf(clblas_st_msg,             \
                      sizeof(clblas_st_msg),     \
@@ -65,4 +69,5 @@ static const char * _clblasGetResultString(clblasStatus st)
             AF_ERROR(clblas_st_msg,             \
                      AF_ERR_INTERNAL);          \
         }                                       \
+        gCLBlasMutex.unlock();                  \
     } while(0)
diff --git a/src/backend/opencl/platform.cpp b/src/backend/opencl/platform.cpp
index 24bdefc2ea..a1646eaa76 100644
--- a/src/backend/opencl/platform.cpp
+++ b/src/backend/opencl/platform.cpp
@@ -761,6 +761,12 @@ DeviceManager::~DeviceManager()
     // clfftTeardown() causes a "Pure Virtual Function Called" crash on
     // Windows for Intel devices. This causes tests to fail.
     clfftTeardown();
+#endif
+#ifndef OS_WIN
+    //TODO: FIXME:
+    // clblasTeardown() causes a "Pure Virtual Function Called" crash on
+    // Windows for Intel devices. This causes tests to fail.
+    clblasTeardown();
 #endif
     delete memManager.release();
     delete pinnedMemManager.release();
diff --git a/test/threading.cpp b/test/threading.cpp
index 181b1270b3..9c044ae792 100644
--- a/test/threading.cpp
+++ b/test/threading.cpp
@@ -616,8 +616,6 @@ TEST(Threading, BLAS)
             tests[testId].join();
 }
 
-#if !defined(AF_OPENCL)
-
 #define SOLVE_LU_TESTS(T, eps)                                                                          \
     tests.emplace_back(solveLUTester<T>, 1000, 100, eps, nextTargetDeviceId()%numDevices);              \
     tests.emplace_back(solveLUTester<T>, 2048, 512, eps, nextTargetDeviceId()%numDevices);              \
@@ -692,5 +690,3 @@ TEST(Threading, Sparse)
         if (tests[testId].joinable())
             tests[testId].join();
 }
-
-#endif

From ed5bc29e29beb976b494ad828337b3a3dee4c69e Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Sat, 22 Apr 2017 23:52:33 +0530
Subject: [PATCH 66/82] Serialize clblast(blas) calls

Blas in OpenCL backend was failing in multiple threads scenario.
Upon discussion with pavan, blas calls are serialized using a
global mutex. This is a temporary fix.
---
 src/backend/opencl/err_clblas.hpp  | 2 +-
 src/backend/opencl/err_clblast.hpp | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/backend/opencl/err_clblas.hpp b/src/backend/opencl/err_clblas.hpp
index 768ead7856..1440582e2c 100644
--- a/src/backend/opencl/err_clblas.hpp
+++ b/src/backend/opencl/err_clblas.hpp
@@ -57,6 +57,7 @@ static std::recursive_mutex gCLBlasMutex;
 #define CLBLAS_CHECK(fn) do {                   \
         gCLBlasMutex.lock();                    \
         clblasStatus _clblas_st = fn;           \
+        gCLBlasMutex.unlock();                  \
         if (_clblas_st != clblasSuccess) {      \
             char clblas_st_msg[1024];           \
             snprintf(clblas_st_msg,             \
@@ -69,5 +70,4 @@ static std::recursive_mutex gCLBlasMutex;
             AF_ERROR(clblas_st_msg,             \
                      AF_ERR_INTERNAL);          \
         }                                       \
-        gCLBlasMutex.unlock();                  \
     } while(0)
diff --git a/src/backend/opencl/err_clblast.hpp b/src/backend/opencl/err_clblast.hpp
index fae1722251..c02d600d49 100644
--- a/src/backend/opencl/err_clblast.hpp
+++ b/src/backend/opencl/err_clblast.hpp
@@ -80,8 +80,12 @@ static const char * _clblastGetResultString(clblast::StatusCode st)
     return "Unknown error";
 }
 
+static std::recursive_mutex gCLBlastMutex;
+
 #define CLBLAST_CHECK(fn) do {                              \
+        gCLBlastMutex.lock();                               \
         clblast::StatusCode _clblast_st = fn;               \
+        gCLBlastMutex.unlock();                             \
         if (_clblast_st != clblast::StatusCode::kSuccess) { \
             char clblast_st_msg[1024];                      \
             snprintf(clblast_st_msg,                        \

From 6727ab654a9b869bec567a4c3b6b48c3e658cbf9 Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Sat, 22 Apr 2017 23:56:47 +0530
Subject: [PATCH 67/82] Reduce memory footprint of solve/sparse multithreaded
 tests

Reduced the memory usage of multi-threaded tests of sparse and
solve to avoid out of memory exception on low memory GPUs.
---
 test/sparse_common.hpp | 17 ++++++++++++++---
 test/threading.cpp     | 33 +++++++--------------------------
 2 files changed, 21 insertions(+), 29 deletions(-)

diff --git a/test/sparse_common.hpp b/test/sparse_common.hpp
index 57705d3ca0..e8a8efc32e 100644
--- a/test/sparse_common.hpp
+++ b/test/sparse_common.hpp
@@ -68,8 +68,12 @@ double calc_norm(af::array lhs, af::array rhs)
 }
 
 template<typename T>
-void sparseTester(const int m, const int n, const int k, int factor, double eps)
+void sparseTester(const int m, const int n, const int k, int factor, double eps,
+                  int targetDevice=-1)
 {
+    if (targetDevice>=0)
+        af::setDevice(targetDevice);
+
     af::deviceGC();
 
     if (noDoubleTests<T>()) return;
@@ -99,8 +103,12 @@ void sparseTester(const int m, const int n, const int k, int factor, double eps)
 }
 
 template<typename T>
-void sparseTransposeTester(const int m, const int n, const int k, int factor, double eps)
+void sparseTransposeTester(const int m, const int n, const int k, int factor, double eps,
+                           int targetDevice=-1)
 {
+    if (targetDevice>=0)
+        af::setDevice(targetDevice);
+
     af::deviceGC();
 
     if (noDoubleTests<T>()) return;
@@ -135,8 +143,11 @@ void sparseTransposeTester(const int m, const int n, const int k, int factor, do
 }
 
 template<typename T>
-void convertCSR(const int M, const int N, const float ratio)
+void convertCSR(const int M, const int N, const float ratio, int targetDevice=-1)
 {
+    if (targetDevice>=0)
+        af::setDevice(targetDevice);
+
     if (noDoubleTests<T>()) return;
 #if 1
     af::array a = cpu_randu<T>(af::dim4(M, N));
diff --git a/test/threading.cpp b/test/threading.cpp
index 9c044ae792..aa427f72b6 100644
--- a/test/threading.cpp
+++ b/test/threading.cpp
@@ -618,25 +618,12 @@ TEST(Threading, BLAS)
 
 #define SOLVE_LU_TESTS(T, eps)                                                                          \
     tests.emplace_back(solveLUTester<T>, 1000, 100, eps, nextTargetDeviceId()%numDevices);              \
-    tests.emplace_back(solveLUTester<T>, 2048, 512, eps, nextTargetDeviceId()%numDevices);              \
-    std::this_thread::sleep_for(std::chrono::seconds(2));   \
     tests.emplace_back(solveTriangleTester<T>, 1000, 100, true, eps, nextTargetDeviceId()%numDevices);  \
-    tests.emplace_back(solveTriangleTester<T>, 2048, 512, true, eps, nextTargetDeviceId()%numDevices);  \
-    std::this_thread::sleep_for(std::chrono::seconds(2));   \
     tests.emplace_back(solveTriangleTester<T>, 1000, 100, false, eps, nextTargetDeviceId()%numDevices); \
-    tests.emplace_back(solveTriangleTester<T>, 2048, 512, false, eps, nextTargetDeviceId()%numDevices); \
-    std::this_thread::sleep_for(std::chrono::seconds(2));   \
     tests.emplace_back(solveTester<T>, 1000, 1000, 100, eps, nextTargetDeviceId()%numDevices);          \
-    tests.emplace_back(solveTester<T>, 2048, 2048, 512, eps, nextTargetDeviceId()%numDevices);          \
-    std::this_thread::sleep_for(std::chrono::seconds(2));   \
     tests.emplace_back(solveTester<T>, 800, 1000, 200, eps, nextTargetDeviceId()%numDevices);           \
-    tests.emplace_back(solveTester<T>, 1536, 2048, 400, eps, nextTargetDeviceId()%numDevices);          \
-    std::this_thread::sleep_for(std::chrono::seconds(2));   \
     tests.emplace_back(solveTester<T>, 800, 600, 64, eps, nextTargetDeviceId()%numDevices);             \
-    tests.emplace_back(solveTester<T>, 1536, 1024, 1, eps, nextTargetDeviceId()%numDevices);
 
-// Added 2s sleep for every two test threads to make sure
-// we are not running out of memory.
 TEST(Threading, SolveDense)
 {
     cleanSlate(); // Clean up everything done so far
@@ -659,19 +646,13 @@ TEST(Threading, SolveDense)
 #undef SOLVE_LU_TESTS
 
 #define SPARSE_TESTS(T, eps)                                                    \
-        tests.emplace_back(sparseTester<T>, 1000, 1000, 100, 5, eps);           \
-        tests.emplace_back(sparseTester<T>, 2048, 1024, 512, 3, eps);           \
-        tests.emplace_back(sparseTester<T>, 500, 1000, 250, 1, eps);            \
-        tests.emplace_back(sparseTester<T>, 625, 1331, 1, 2, eps);              \
-        tests.emplace_back(sparseTransposeTester<T>, 625, 1331, 1, 2, eps);     \
-        tests.emplace_back(sparseTransposeTester<T>, 1000, 1000, 100, 5, eps);  \
-        tests.emplace_back(sparseTransposeTester<T>, 2048, 1024, 512, 3, eps);  \
-        tests.emplace_back(sparseTransposeTester<T>, 453, 751, 397, 1, eps);    \
-        tests.emplace_back(convertCSR<T>, 2345, 5678, 0.5);                     \
-        std::this_thread::sleep_for(std::chrono::seconds(5));
-
-// Added 2s sleep for every two test threads to make sure
-// we are not running out of memory.
+  tests.emplace_back(sparseTester<T>, 1000, 1000, 100, 5, eps, nextTargetDeviceId()%numDevices);  \
+  tests.emplace_back(sparseTester<T>, 500, 1000, 250, 1, eps, nextTargetDeviceId()%numDevices);   \
+  tests.emplace_back(sparseTester<T>, 625, 1331, 1, 2, eps, nextTargetDeviceId()%numDevices);     \
+  tests.emplace_back(sparseTransposeTester<T>, 625, 1331, 1, 2, eps, nextTargetDeviceId()%numDevices); \
+  tests.emplace_back(sparseTransposeTester<T>, 453, 751, 397, 1, eps, nextTargetDeviceId()%numDevices);\
+  tests.emplace_back(convertCSR<T>, 2345, 5678, 0.5, nextTargetDeviceId()%numDevices);
+
 TEST(Threading, Sparse)
 {
     cleanSlate(); // Clean up everything done so far

From 53e1fd8cdb2bfe2ec37503cc42bc85ea30c8c029 Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Sun, 23 Apr 2017 00:06:47 +0530
Subject: [PATCH 68/82] Disable Solve/Sparse multi-threaded tests for OpenCL

---
 test/threading.cpp | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/test/threading.cpp b/test/threading.cpp
index aa427f72b6..5301567187 100644
--- a/test/threading.cpp
+++ b/test/threading.cpp
@@ -616,6 +616,8 @@ TEST(Threading, BLAS)
             tests[testId].join();
 }
 
+#if !defined(AF_OPENCL)
+
 #define SOLVE_LU_TESTS(T, eps)                                                                          \
     tests.emplace_back(solveLUTester<T>, 1000, 100, eps, nextTargetDeviceId()%numDevices);              \
     tests.emplace_back(solveTriangleTester<T>, 1000, 100, true, eps, nextTargetDeviceId()%numDevices);  \
@@ -671,3 +673,5 @@ TEST(Threading, Sparse)
         if (tests[testId].joinable())
             tests[testId].join();
 }
+
+#endif

From bdb3d6ef3b2af5d0703b2f01dc93aae6040d5da0 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Mon, 24 Apr 2017 16:37:37 -0400
Subject: [PATCH 69/82] Disable double tests for devices that do not support
 doubles

---
 test/threading.cpp | 112 ++++++++++++++++++++++++++++-----------------
 1 file changed, 69 insertions(+), 43 deletions(-)

diff --git a/test/threading.cpp b/test/threading.cpp
index 5301567187..5b1cdc14b0 100644
--- a/test/threading.cpp
+++ b/test/threading.cpp
@@ -396,23 +396,33 @@ TEST(Threading, FFT_R2C)
 
     // Real to complex transforms
     INSTANTIATE_TEST(fft ,  R2C_Float, false,  float,  cfloat, string(TEST_DIR"/signal/fft_r2c.test") );
-    INSTANTIATE_TEST(fft , R2C_Double, false, double, cdouble, string(TEST_DIR"/signal/fft_r2c.test") );
     INSTANTIATE_TEST(fft2,  R2C_Float, false,  float,  cfloat, string(TEST_DIR"/signal/fft2_r2c.test"));
-    INSTANTIATE_TEST(fft2, R2C_Double, false, double, cdouble, string(TEST_DIR"/signal/fft2_r2c.test"));
     INSTANTIATE_TEST(fft3,  R2C_Float, false,  float,  cfloat, string(TEST_DIR"/signal/fft3_r2c.test"));
-    INSTANTIATE_TEST(fft3, R2C_Double, false, double, cdouble, string(TEST_DIR"/signal/fft3_r2c.test"));
 
     // Factors 7, 11, 13
     INSTANTIATE_TEST(fft , R2C_Float_7_11_13 , false, float  , cfloat , string(TEST_DIR"/signal/fft_r2c_7_11_13.test") );
-    INSTANTIATE_TEST(fft , R2C_Double_7_11_13, false, double , cdouble, string(TEST_DIR"/signal/fft_r2c_7_11_13.test") );
     INSTANTIATE_TEST(fft2, R2C_Float_7_11_13 , false, float  , cfloat , string(TEST_DIR"/signal/fft2_r2c_7_11_13.test") );
-    INSTANTIATE_TEST(fft2, R2C_Double_7_11_13, false, double , cdouble, string(TEST_DIR"/signal/fft2_r2c_7_11_13.test") );
     INSTANTIATE_TEST(fft3, R2C_Float_7_11_13 , false, float  , cfloat , string(TEST_DIR"/signal/fft3_r2c_7_11_13.test") );
-    INSTANTIATE_TEST(fft3, R2C_Double_7_11_13, false, double , cdouble, string(TEST_DIR"/signal/fft3_r2c_7_11_13.test") );
 
     // transforms on padded and truncated arrays
     INSTANTIATE_TEST_TP(fft2,  R2C_Float_Trunc, false,  float,  cfloat, string(TEST_DIR"/signal/fft2_r2c_trunc.test"), 16, 16);
-    INSTANTIATE_TEST_TP(fft2, R2C_Double_Trunc, false, double, cdouble, string(TEST_DIR"/signal/fft2_r2c_trunc.test"), 16, 16);
+
+
+    if (noDoubleTests<double>()) {
+        // Real to complex transforms
+        INSTANTIATE_TEST(fft , R2C_Double, false, double, cdouble, string(TEST_DIR"/signal/fft_r2c.test") );
+        INSTANTIATE_TEST(fft2, R2C_Double, false, double, cdouble, string(TEST_DIR"/signal/fft2_r2c.test"));
+        INSTANTIATE_TEST(fft3, R2C_Double, false, double, cdouble, string(TEST_DIR"/signal/fft3_r2c.test"));
+
+        // Factors 7, 11, 13
+        INSTANTIATE_TEST(fft , R2C_Double_7_11_13, false, double , cdouble, string(TEST_DIR"/signal/fft_r2c_7_11_13.test") );
+        INSTANTIATE_TEST(fft2, R2C_Double_7_11_13, false, double , cdouble, string(TEST_DIR"/signal/fft2_r2c_7_11_13.test") );
+        INSTANTIATE_TEST(fft3, R2C_Double_7_11_13, false, double , cdouble, string(TEST_DIR"/signal/fft3_r2c_7_11_13.test") );
+
+        // transforms on padded and truncated arrays
+        INSTANTIATE_TEST_TP(fft2, R2C_Double_Trunc, false, double, cdouble, string(TEST_DIR"/signal/fft2_r2c_trunc.test"), 16, 16);
+    }
+
 
     for (size_t testId=0; testId<tests.size(); ++testId)
         if (tests[testId].joinable())
@@ -430,31 +440,38 @@ TEST(Threading, FFT_C2C)
 
     // complex to complex transforms
     INSTANTIATE_TEST(fft ,  C2C_Float, false,  cfloat,  cfloat, string(TEST_DIR"/signal/fft_c2c.test") );
-    INSTANTIATE_TEST(fft , C2C_Double, false, cdouble, cdouble, string(TEST_DIR"/signal/fft_c2c.test") );
     INSTANTIATE_TEST(fft2,  C2C_Float, false,  cfloat,  cfloat, string(TEST_DIR"/signal/fft2_c2c.test"));
-    INSTANTIATE_TEST(fft2, C2C_Double, false, cdouble, cdouble, string(TEST_DIR"/signal/fft2_c2c.test"));
     INSTANTIATE_TEST(fft3,  C2C_Float, false,  cfloat,  cfloat, string(TEST_DIR"/signal/fft3_c2c.test"));
-    INSTANTIATE_TEST(fft3, C2C_Double, false, cdouble, cdouble, string(TEST_DIR"/signal/fft3_c2c.test"));
 
     INSTANTIATE_TEST(fft , C2C_Float_7_11_13 , false, cfloat  , cfloat , string(TEST_DIR"/signal/fft_c2c_7_11_13.test") );
-    INSTANTIATE_TEST(fft , C2C_Double_7_11_13, false, cdouble , cdouble, string(TEST_DIR"/signal/fft_c2c_7_11_13.test") );
     INSTANTIATE_TEST(fft2, C2C_Float_7_11_13 , false, cfloat  , cfloat , string(TEST_DIR"/signal/fft2_c2c_7_11_13.test") );
-    INSTANTIATE_TEST(fft2, C2C_Double_7_11_13, false, cdouble , cdouble, string(TEST_DIR"/signal/fft2_c2c_7_11_13.test") );
     INSTANTIATE_TEST(fft3, C2C_Float_7_11_13 , false, cfloat  , cfloat , string(TEST_DIR"/signal/fft3_c2c_7_11_13.test") );
-    INSTANTIATE_TEST(fft3, C2C_Double_7_11_13, false, cdouble , cdouble, string(TEST_DIR"/signal/fft3_c2c_7_11_13.test") );
 
     // transforms on padded and truncated arrays
     INSTANTIATE_TEST_TP(fft2,  C2C_Float_Pad, false,  cfloat,  cfloat, string(TEST_DIR"/signal/fft2_c2c_pad.test"), 16, 16);
-    INSTANTIATE_TEST_TP(fft2, C2C_Double_Pad, false, cdouble, cdouble, string(TEST_DIR"/signal/fft2_c2c_pad.test"), 16, 16);
 
     // inverse transforms
     // complex to complex transforms
     INSTANTIATE_TEST(ifft ,  C2C_Float, true,  cfloat,  cfloat, string(TEST_DIR"/signal/ifft_c2c.test") );
-    INSTANTIATE_TEST(ifft , C2C_Double, true, cdouble, cdouble, string(TEST_DIR"/signal/ifft_c2c.test") );
     INSTANTIATE_TEST(ifft2,  C2C_Float, true,  cfloat,  cfloat, string(TEST_DIR"/signal/ifft2_c2c.test"));
-    INSTANTIATE_TEST(ifft2, C2C_Double, true, cdouble, cdouble, string(TEST_DIR"/signal/ifft2_c2c.test"));
     INSTANTIATE_TEST(ifft3,  C2C_Float, true,  cfloat,  cfloat, string(TEST_DIR"/signal/ifft3_c2c.test"));
-    INSTANTIATE_TEST(ifft3, C2C_Double, true, cdouble, cdouble, string(TEST_DIR"/signal/ifft3_c2c.test"));
+
+
+    if (noDoubleTests<double>()) {
+        INSTANTIATE_TEST(fft , C2C_Double, false, cdouble, cdouble, string(TEST_DIR"/signal/fft_c2c.test") );
+        INSTANTIATE_TEST(fft2, C2C_Double, false, cdouble, cdouble, string(TEST_DIR"/signal/fft2_c2c.test"));
+        INSTANTIATE_TEST(fft3, C2C_Double, false, cdouble, cdouble, string(TEST_DIR"/signal/fft3_c2c.test"));
+
+        INSTANTIATE_TEST(fft , C2C_Double_7_11_13, false, cdouble , cdouble, string(TEST_DIR"/signal/fft_c2c_7_11_13.test") );
+        INSTANTIATE_TEST(fft2, C2C_Double_7_11_13, false, cdouble , cdouble, string(TEST_DIR"/signal/fft2_c2c_7_11_13.test") );
+        INSTANTIATE_TEST(fft3, C2C_Double_7_11_13, false, cdouble , cdouble, string(TEST_DIR"/signal/fft3_c2c_7_11_13.test") );
+
+        INSTANTIATE_TEST_TP(fft2, C2C_Double_Pad, false, cdouble, cdouble, string(TEST_DIR"/signal/fft2_c2c_pad.test"), 16, 16);
+
+        INSTANTIATE_TEST(ifft , C2C_Double, true, cdouble, cdouble, string(TEST_DIR"/signal/ifft_c2c.test") );
+        INSTANTIATE_TEST(ifft2, C2C_Double, true, cdouble, cdouble, string(TEST_DIR"/signal/ifft2_c2c.test"));
+        INSTANTIATE_TEST(ifft3, C2C_Double, true, cdouble, cdouble, string(TEST_DIR"/signal/ifft3_c2c.test"));
+    }
 
     for (size_t testId=0; testId<tests.size(); ++testId)
         if (tests[testId].joinable())
@@ -472,51 +489,54 @@ TEST(Threading, FFT_ALL)
 
     // Real to complex transforms
     INSTANTIATE_TEST(fft ,  R2C_Float, false,  float,  cfloat, string(TEST_DIR"/signal/fft_r2c.test") );
-    INSTANTIATE_TEST(fft , R2C_Double, false, double, cdouble, string(TEST_DIR"/signal/fft_r2c.test") );
     INSTANTIATE_TEST(fft2,  R2C_Float, false,  float,  cfloat, string(TEST_DIR"/signal/fft2_r2c.test"));
-    INSTANTIATE_TEST(fft2, R2C_Double, false, double, cdouble, string(TEST_DIR"/signal/fft2_r2c.test"));
     INSTANTIATE_TEST(fft3,  R2C_Float, false,  float,  cfloat, string(TEST_DIR"/signal/fft3_r2c.test"));
-    INSTANTIATE_TEST(fft3, R2C_Double, false, double, cdouble, string(TEST_DIR"/signal/fft3_r2c.test"));
 
     // Factors 7, 11, 13
     INSTANTIATE_TEST(fft , R2C_Float_7_11_13 , false, float  , cfloat , string(TEST_DIR"/signal/fft_r2c_7_11_13.test") );
-    INSTANTIATE_TEST(fft , R2C_Double_7_11_13, false, double , cdouble, string(TEST_DIR"/signal/fft_r2c_7_11_13.test") );
     INSTANTIATE_TEST(fft2, R2C_Float_7_11_13 , false, float  , cfloat , string(TEST_DIR"/signal/fft2_r2c_7_11_13.test") );
-    INSTANTIATE_TEST(fft2, R2C_Double_7_11_13, false, double , cdouble, string(TEST_DIR"/signal/fft2_r2c_7_11_13.test") );
     INSTANTIATE_TEST(fft3, R2C_Float_7_11_13 , false, float  , cfloat , string(TEST_DIR"/signal/fft3_r2c_7_11_13.test") );
-    INSTANTIATE_TEST(fft3, R2C_Double_7_11_13, false, double , cdouble, string(TEST_DIR"/signal/fft3_r2c_7_11_13.test") );
 
     // transforms on padded and truncated arrays
     INSTANTIATE_TEST_TP(fft2,  R2C_Float_Trunc, false,  float,  cfloat, string(TEST_DIR"/signal/fft2_r2c_trunc.test"), 16, 16);
-    INSTANTIATE_TEST_TP(fft2, R2C_Double_Trunc, false, double, cdouble, string(TEST_DIR"/signal/fft2_r2c_trunc.test"), 16, 16);
 
     // complex to complex transforms
     INSTANTIATE_TEST(fft ,  C2C_Float, false,  cfloat,  cfloat, string(TEST_DIR"/signal/fft_c2c.test") );
-    INSTANTIATE_TEST(fft , C2C_Double, false, cdouble, cdouble, string(TEST_DIR"/signal/fft_c2c.test") );
     INSTANTIATE_TEST(fft2,  C2C_Float, false,  cfloat,  cfloat, string(TEST_DIR"/signal/fft2_c2c.test"));
-    INSTANTIATE_TEST(fft2, C2C_Double, false, cdouble, cdouble, string(TEST_DIR"/signal/fft2_c2c.test"));
     INSTANTIATE_TEST(fft3,  C2C_Float, false,  cfloat,  cfloat, string(TEST_DIR"/signal/fft3_c2c.test"));
-    INSTANTIATE_TEST(fft3, C2C_Double, false, cdouble, cdouble, string(TEST_DIR"/signal/fft3_c2c.test"));
 
     INSTANTIATE_TEST(fft , C2C_Float_7_11_13 , false, cfloat  , cfloat , string(TEST_DIR"/signal/fft_c2c_7_11_13.test") );
-    INSTANTIATE_TEST(fft , C2C_Double_7_11_13, false, cdouble , cdouble, string(TEST_DIR"/signal/fft_c2c_7_11_13.test") );
     INSTANTIATE_TEST(fft2, C2C_Float_7_11_13 , false, cfloat  , cfloat , string(TEST_DIR"/signal/fft2_c2c_7_11_13.test") );
-    INSTANTIATE_TEST(fft2, C2C_Double_7_11_13, false, cdouble , cdouble, string(TEST_DIR"/signal/fft2_c2c_7_11_13.test") );
     INSTANTIATE_TEST(fft3, C2C_Float_7_11_13 , false, cfloat  , cfloat , string(TEST_DIR"/signal/fft3_c2c_7_11_13.test") );
-    INSTANTIATE_TEST(fft3, C2C_Double_7_11_13, false, cdouble , cdouble, string(TEST_DIR"/signal/fft3_c2c_7_11_13.test") );
 
     // transforms on padded and truncated arrays
     INSTANTIATE_TEST_TP(fft2,  C2C_Float_Pad, false,  cfloat,  cfloat, string(TEST_DIR"/signal/fft2_c2c_pad.test"), 16, 16);
-    INSTANTIATE_TEST_TP(fft2, C2C_Double_Pad, false, cdouble, cdouble, string(TEST_DIR"/signal/fft2_c2c_pad.test"), 16, 16);
 
     // inverse transforms
     // complex to complex transforms
     INSTANTIATE_TEST(ifft ,  C2C_Float, true,  cfloat,  cfloat, string(TEST_DIR"/signal/ifft_c2c.test") );
-    INSTANTIATE_TEST(ifft , C2C_Double, true, cdouble, cdouble, string(TEST_DIR"/signal/ifft_c2c.test") );
     INSTANTIATE_TEST(ifft2,  C2C_Float, true,  cfloat,  cfloat, string(TEST_DIR"/signal/ifft2_c2c.test"));
-    INSTANTIATE_TEST(ifft2, C2C_Double, true, cdouble, cdouble, string(TEST_DIR"/signal/ifft2_c2c.test"));
     INSTANTIATE_TEST(ifft3,  C2C_Float, true,  cfloat,  cfloat, string(TEST_DIR"/signal/ifft3_c2c.test"));
-    INSTANTIATE_TEST(ifft3, C2C_Double, true, cdouble, cdouble, string(TEST_DIR"/signal/ifft3_c2c.test"));
+
+    if (noDoubleTests<double>()) {
+        INSTANTIATE_TEST(fft , R2C_Double, false, double, cdouble, string(TEST_DIR"/signal/fft_r2c.test") );
+        INSTANTIATE_TEST(fft2, R2C_Double, false, double, cdouble, string(TEST_DIR"/signal/fft2_r2c.test"));
+        INSTANTIATE_TEST(fft3, R2C_Double, false, double, cdouble, string(TEST_DIR"/signal/fft3_r2c.test"));
+        INSTANTIATE_TEST(fft , R2C_Double_7_11_13, false, double , cdouble, string(TEST_DIR"/signal/fft_r2c_7_11_13.test") );
+        INSTANTIATE_TEST(fft2, R2C_Double_7_11_13, false, double , cdouble, string(TEST_DIR"/signal/fft2_r2c_7_11_13.test") );
+        INSTANTIATE_TEST(fft3, R2C_Double_7_11_13, false, double , cdouble, string(TEST_DIR"/signal/fft3_r2c_7_11_13.test") );
+        INSTANTIATE_TEST_TP(fft2, R2C_Double_Trunc, false, double, cdouble, string(TEST_DIR"/signal/fft2_r2c_trunc.test"), 16, 16);
+        INSTANTIATE_TEST(fft , C2C_Double, false, cdouble, cdouble, string(TEST_DIR"/signal/fft_c2c.test") );
+        INSTANTIATE_TEST(fft2, C2C_Double, false, cdouble, cdouble, string(TEST_DIR"/signal/fft2_c2c.test"));
+        INSTANTIATE_TEST(fft3, C2C_Double, false, cdouble, cdouble, string(TEST_DIR"/signal/fft3_c2c.test"));
+        INSTANTIATE_TEST(fft , C2C_Double_7_11_13, false, cdouble , cdouble, string(TEST_DIR"/signal/fft_c2c_7_11_13.test") );
+        INSTANTIATE_TEST(fft2, C2C_Double_7_11_13, false, cdouble , cdouble, string(TEST_DIR"/signal/fft2_c2c_7_11_13.test") );
+        INSTANTIATE_TEST(fft3, C2C_Double_7_11_13, false, cdouble , cdouble, string(TEST_DIR"/signal/fft3_c2c_7_11_13.test") );
+        INSTANTIATE_TEST_TP(fft2, C2C_Double_Pad, false, cdouble, cdouble, string(TEST_DIR"/signal/fft2_c2c_pad.test"), 16, 16);
+        INSTANTIATE_TEST(ifft , C2C_Double, true, cdouble, cdouble, string(TEST_DIR"/signal/ifft_c2c.test") );
+        INSTANTIATE_TEST(ifft2, C2C_Double, true, cdouble, cdouble, string(TEST_DIR"/signal/ifft2_c2c.test"));
+        INSTANTIATE_TEST(ifft3, C2C_Double, true, cdouble, cdouble, string(TEST_DIR"/signal/ifft3_c2c.test"));
+    }
 
     for (size_t testId=0; testId<tests.size(); ++testId)
         if (tests[testId].joinable())
@@ -587,7 +607,7 @@ void cppMatMulCheck(int targetDevice, string TestFile)
     }
 }
 
-#define TEST_FOR_TYPE(TypeName)                                   \
+#define TEST_FOR_TYPE(TypeName)                                         \
     tests.emplace_back(cppMatMulCheck<TypeName, false>,                             \
             nextTargetDeviceId()%numDevices, TEST_DIR "/blas/Basic.test");          \
     tests.emplace_back(cppMatMulCheck<TypeName, false>,                             \
@@ -608,8 +628,11 @@ TEST(Threading, BLAS)
 
     TEST_FOR_TYPE(      float);
     TEST_FOR_TYPE( af::cfloat);
-    TEST_FOR_TYPE(     double);
-    TEST_FOR_TYPE(af::cdouble);
+
+    if (noDoubleTests<double>()) {
+        TEST_FOR_TYPE(     double);
+        TEST_FOR_TYPE(af::cdouble);
+    }
 
     for (size_t testId=0; testId<tests.size(); ++testId)
         if (tests[testId].joinable())
@@ -636,9 +659,11 @@ TEST(Threading, SolveDense)
     ASSERT_EQ(AF_SUCCESS, af_get_device_count(&numDevices));
 
     SOLVE_LU_TESTS(float, 0.01);
-    SOLVE_LU_TESTS(double, 1E-5);
     SOLVE_LU_TESTS(cfloat, 0.01);
-    SOLVE_LU_TESTS(cdouble, 1E-5);
+    if (noDoubleTests<double>()) {
+        SOLVE_LU_TESTS(double, 1E-5);
+        SOLVE_LU_TESTS(cdouble, 1E-5);
+    }
 
     for (size_t testId=0; testId<tests.size(); ++testId)
         if (tests[testId].joinable())
@@ -646,6 +671,7 @@ TEST(Threading, SolveDense)
 }
 
 #undef SOLVE_LU_TESTS
+#endif
 
 #define SPARSE_TESTS(T, eps)                                                    \
   tests.emplace_back(sparseTester<T>, 1000, 1000, 100, 5, eps, nextTargetDeviceId()%numDevices);  \
@@ -665,13 +691,13 @@ TEST(Threading, Sparse)
     ASSERT_EQ(AF_SUCCESS, af_get_device_count(&numDevices));
 
     SPARSE_TESTS(  float, 1E-3);
-    SPARSE_TESTS( double, 1E-5);
     SPARSE_TESTS( cfloat, 1E-3);
-    SPARSE_TESTS(cdouble, 1E-5);
+    if (noDoubleTests<double>()) {
+        SPARSE_TESTS( double, 1E-5);
+        SPARSE_TESTS(cdouble, 1E-5);
+    }
 
     for (size_t testId=0; testId<tests.size(); ++testId)
         if (tests[testId].joinable())
             tests[testId].join();
 }
-
-#endif

From da684f07d185a153193ca5c858d7c9f9e253bbd7 Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Tue, 25 Apr 2017 21:40:04 +0530
Subject: [PATCH 70/82] Use a pool of cuda streams in round robin fashion for
 threads

Each thread will have a separate stream, although if too many threads
are launched i.e. greater than the pool size, then existing set of
streams are just used up in round robin fashion.
---
 src/backend/cuda/platform.cpp | 36 +++++++++++++++++++++++------------
 src/backend/cuda/platform.hpp |  9 ++++++++-
 2 files changed, 32 insertions(+), 13 deletions(-)

diff --git a/src/backend/cuda/platform.cpp b/src/backend/cuda/platform.cpp
index 89cc827758..3fbad720a4 100644
--- a/src/backend/cuda/platform.cpp
+++ b/src/backend/cuda/platform.cpp
@@ -316,17 +316,23 @@ int getDeviceIdFromNativeId(int nativeId)
     return devId;
 }
 
-cudaStream_t getStream(int device)
+cudaStream_t DeviceManager::nextAvailableStream(int device)
 {
-    static std::once_flag streamInitFlags[DeviceManager::MAX_DEVICES];
+    common::lock_guard_t lock(poolCounterMutexes[device]);
+
+    unsigned oldPoolId = nextPoolCounter[device];
 
-    std::call_once(streamInitFlags[device],
-            [device]() {
-                DeviceManager& inst = DeviceManager::getInstance();
-                CUDA_CHECK(cudaStreamCreate( & (inst.streams[device]) ));
-            });
+    nextPoolCounter[device] = (oldPoolId+1) % streamPoolCluster[device].size();
 
-    return DeviceManager::getInstance().streams[device];
+    return streamPoolCluster[device][oldPoolId];
+}
+
+cudaStream_t getStream(int device)
+{
+    static thread_local cudaStream_t myDefaultStream =
+        DeviceManager::getInstance().nextAvailableStream(device);
+
+    return myDefaultStream;
 }
 
 cudaStream_t getActiveStream()
@@ -508,10 +514,16 @@ DeviceManager::DeviceManager()
 
     sortDevices();
 
-    // Initialize all streams to 0.
-    // Streams will be created in setActiveDevice()
-    for(int i = 0; i < (int)MAX_DEVICES; i++)
-        streams[i] = (cudaStream_t)0;
+    // Initialize stream pools for all devices.
+    for (unsigned c=0; c < streamPoolCluster.size(); ++c) {
+        streamPoolCluster[c].resize(MAX_SIZE_STREAM_POOL, static_cast<cudaStream_t>(0));
+        for (unsigned p=0; p<MAX_SIZE_STREAM_POOL; ++p) {
+            cudaStream_t temp = 0;
+            CUDA_CHECK(cudaStreamCreate(&temp));
+            streamPoolCluster[c][p] = temp;
+        }
+    }
+    nextPoolCounter.fill(0);
 
     std::string deviceENV = getEnvVar("AF_CUDA_DEFAULT_DEVICE");
     if(deviceENV.empty()) {
diff --git a/src/backend/cuda/platform.hpp b/src/backend/cuda/platform.hpp
index b9ac3df401..f93b8c6b94 100644
--- a/src/backend/cuda/platform.hpp
+++ b/src/backend/cuda/platform.hpp
@@ -96,6 +96,8 @@ class DeviceManager
 {
     public:
         static const unsigned MAX_DEVICES = 16;
+        static const unsigned MAX_SIZE_STREAM_POOL = 16;
+        typedef std::vector<cudaStream_t> StreamPool;
 
 #if defined(WITH_GRAPHICS)
         static bool checkGraphicsInteropCapability();
@@ -152,8 +154,13 @@ class DeviceManager
 
         int setActiveDevice(int device, int native = -1);
 
+        cudaStream_t nextAvailableStream(int device);
+
         int nDevices;
-        cudaStream_t streams[MAX_DEVICES];
+
+        std::array<     StreamPool, MAX_DEVICES> streamPoolCluster;
+        std::array<       unsigned, MAX_DEVICES> nextPoolCounter;
+        std::array<common::mutex_t, MAX_DEVICES> poolCounterMutexes;
 
         std::unique_ptr<MemoryManager> memManager;
 

From 3c765754fc26d5dc47176d0ea407551ec3ea7504 Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Tue, 25 Apr 2017 22:05:31 +0530
Subject: [PATCH 71/82] Typo fix in accum/af_accum fn documentation

---
 docs/details/algorithm.dox | 2 +-
 include/af/algorithm.h     | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/details/algorithm.dox b/docs/details/algorithm.dox
index 27ef55ef48..6e6f3b8cde 100644
--- a/docs/details/algorithm.dox
+++ b/docs/details/algorithm.dox
@@ -107,7 +107,7 @@ Return type is u32 for all input types
 
 \ingroup scan_mat
 
-Perform exclusive sum along specified dimension
+Perform inclusive sum along specified dimension
 
 This table defines the return value types for the corresponding input types
 
diff --git a/include/af/algorithm.h b/include/af/algorithm.h
index 39d948fc20..737ef1a3d3 100644
--- a/include/af/algorithm.h
+++ b/include/af/algorithm.h
@@ -306,7 +306,7 @@ namespace af
     template<typename T> void max(T *val, unsigned *idx, const array &in);
 
     /**
-       C++ Interface exclusive sum (cumulative sum) of an array
+       C++ Interface inclusive sum (cumulative sum) of an array
 
        \param[in] in is the input array
        \param[in] dim The dimension along which exclusive sum is performed
@@ -761,7 +761,7 @@ extern "C" {
     AFAPI af_err af_imax_all(double *real, double *imag, unsigned *idx, const af_array in);
 
     /**
-       C Interface exclusive sum (cumulative sum) of an array
+       C Interface inclusive sum (cumulative sum) of an array
 
        \param[out] out will contain exclusive sums of the input
        \param[in] in is the input array

From 4b6a93195470fd2933a0b44f30c85295bb890f9e Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Tue, 25 Apr 2017 18:39:19 -0400
Subject: [PATCH 72/82] Add a memory manager stress test.

---
 test/threading.cpp | 35 ++++++++++++++++++++++++++++++++++-
 1 file changed, 34 insertions(+), 1 deletion(-)

diff --git a/test/threading.cpp b/test/threading.cpp
index 5b1cdc14b0..60d088e79d 100644
--- a/test/threading.cpp
+++ b/test/threading.cpp
@@ -16,6 +16,7 @@
 #include <chrono>
 #include <thread>
 #include <iterator>
+#include <vector>
 #include <testHelpers.hpp>
 #include <solve_common.hpp>
 #include <sparse_common.hpp>
@@ -697,7 +698,39 @@ TEST(Threading, Sparse)
         SPARSE_TESTS(cdouble, 1E-5);
     }
 
-    for (size_t testId=0; testId<tests.size(); ++testId)
+    for (size_t testId=0; testId < tests.size(); ++testId)
         if (tests[testId].joinable())
             tests[testId].join();
 }
+
+TEST(Threading, MemoryManagerStressTest) {
+  vector<std::thread> threads;
+  for (int i = 0; i < THREAD_COUNT; i++) {
+    threads.emplace_back([] {
+        vector<af::array> arrg;
+        int size = 100;
+        int ex_count = 0;
+
+        // Continue until the memory runs out multiple times
+        while (true) {
+          try {
+            // constantly change size of the array allocated
+            size+=10;
+            arrg.push_back(af::randu(size));
+
+            // delete some values intermittently
+            if (!(size%200)) {
+              arrg.erase(std::begin(arrg), std::begin(arrg)+5);
+            }
+          } catch( const af::exception &ex ) {
+            if (ex_count++ > 3) {
+              break;
+            }
+          }
+        }
+      });
+  }
+  for (auto& t : threads) {
+    t.join();
+  }
+}

From fdd2ba032b06ecbae7ca606684cafe874f86a37d Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Wed, 26 Apr 2017 09:56:12 +0530
Subject: [PATCH 73/82] Revert "Use a pool of cuda streams in round robin
 fashion for threads"

This reverts commit da684f07d185a153193ca5c858d7c9f9e253bbd7.
---
 src/backend/cuda/platform.cpp | 36 ++++++++++++-----------------------
 src/backend/cuda/platform.hpp |  9 +--------
 2 files changed, 13 insertions(+), 32 deletions(-)

diff --git a/src/backend/cuda/platform.cpp b/src/backend/cuda/platform.cpp
index 3fbad720a4..89cc827758 100644
--- a/src/backend/cuda/platform.cpp
+++ b/src/backend/cuda/platform.cpp
@@ -316,23 +316,17 @@ int getDeviceIdFromNativeId(int nativeId)
     return devId;
 }
 
-cudaStream_t DeviceManager::nextAvailableStream(int device)
-{
-    common::lock_guard_t lock(poolCounterMutexes[device]);
-
-    unsigned oldPoolId = nextPoolCounter[device];
-
-    nextPoolCounter[device] = (oldPoolId+1) % streamPoolCluster[device].size();
-
-    return streamPoolCluster[device][oldPoolId];
-}
-
 cudaStream_t getStream(int device)
 {
-    static thread_local cudaStream_t myDefaultStream =
-        DeviceManager::getInstance().nextAvailableStream(device);
+    static std::once_flag streamInitFlags[DeviceManager::MAX_DEVICES];
+
+    std::call_once(streamInitFlags[device],
+            [device]() {
+                DeviceManager& inst = DeviceManager::getInstance();
+                CUDA_CHECK(cudaStreamCreate( & (inst.streams[device]) ));
+            });
 
-    return myDefaultStream;
+    return DeviceManager::getInstance().streams[device];
 }
 
 cudaStream_t getActiveStream()
@@ -514,16 +508,10 @@ DeviceManager::DeviceManager()
 
     sortDevices();
 
-    // Initialize stream pools for all devices.
-    for (unsigned c=0; c < streamPoolCluster.size(); ++c) {
-        streamPoolCluster[c].resize(MAX_SIZE_STREAM_POOL, static_cast<cudaStream_t>(0));
-        for (unsigned p=0; p<MAX_SIZE_STREAM_POOL; ++p) {
-            cudaStream_t temp = 0;
-            CUDA_CHECK(cudaStreamCreate(&temp));
-            streamPoolCluster[c][p] = temp;
-        }
-    }
-    nextPoolCounter.fill(0);
+    // Initialize all streams to 0.
+    // Streams will be created in setActiveDevice()
+    for(int i = 0; i < (int)MAX_DEVICES; i++)
+        streams[i] = (cudaStream_t)0;
 
     std::string deviceENV = getEnvVar("AF_CUDA_DEFAULT_DEVICE");
     if(deviceENV.empty()) {
diff --git a/src/backend/cuda/platform.hpp b/src/backend/cuda/platform.hpp
index f93b8c6b94..b9ac3df401 100644
--- a/src/backend/cuda/platform.hpp
+++ b/src/backend/cuda/platform.hpp
@@ -96,8 +96,6 @@ class DeviceManager
 {
     public:
         static const unsigned MAX_DEVICES = 16;
-        static const unsigned MAX_SIZE_STREAM_POOL = 16;
-        typedef std::vector<cudaStream_t> StreamPool;
 
 #if defined(WITH_GRAPHICS)
         static bool checkGraphicsInteropCapability();
@@ -154,13 +152,8 @@ class DeviceManager
 
         int setActiveDevice(int device, int native = -1);
 
-        cudaStream_t nextAvailableStream(int device);
-
         int nDevices;
-
-        std::array<     StreamPool, MAX_DEVICES> streamPoolCluster;
-        std::array<       unsigned, MAX_DEVICES> nextPoolCounter;
-        std::array<common::mutex_t, MAX_DEVICES> poolCounterMutexes;
+        cudaStream_t streams[MAX_DEVICES];
 
         std::unique_ptr<MemoryManager> memManager;
 

From 48482d443ab604bb3cba3a00783262e8aa663196 Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Wed, 26 Apr 2017 17:52:07 +0530
Subject: [PATCH 74/82] Replace new/deleta with std::vector in image processing
 tests

Also, fixes memory leaks in medfilt and morph unit tests
---
 test/bilateral.cpp      | 27 +++++++++++----------------
 test/canny.cpp          | 15 +++++++--------
 test/covariance.cpp     |  7 ++-----
 test/gen_assign.cpp     | 19 ++++++++++---------
 test/gen_index.cpp      | 10 ++++------
 test/histogram.cpp      | 14 +++++---------
 test/hsv_rgb.cpp        | 14 ++++----------
 test/match_template.cpp |  5 ++---
 test/mean.cpp           | 14 ++++----------
 test/meanshift.cpp      | 23 ++++++++++-------------
 test/medfilt.cpp        | 35 ++++++++++++++---------------------
 test/morph.cpp          | 33 ++++++++++++++-------------------
 test/sobel.cpp          | 10 ++++------
 test/stdev.cpp          | 14 ++++----------
 test/susan.cpp          | 29 ++++++++++++-----------------
 test/ycbcr_rgb.cpp      | 14 ++++----------
 16 files changed, 111 insertions(+), 172 deletions(-)

diff --git a/test/bilateral.cpp b/test/bilateral.cpp
index cde330dca4..2362e1c134 100644
--- a/test/bilateral.cpp
+++ b/test/bilateral.cpp
@@ -52,13 +52,13 @@ void bilateralTest(string pTestFile)
 
         ASSERT_EQ(AF_SUCCESS, af_bilateral(&outArray, inArray, 2.25f, 25.56f, isColor));
 
-        T * outData = new T[nElems];
-        ASSERT_EQ(AF_SUCCESS, af_get_data_ptr((void*)outData, outArray));
+        std::vector<T> outData(nElems);
+        ASSERT_EQ(AF_SUCCESS, af_get_data_ptr((void*)outData.data(), outArray));
 
-        T * goldData= new T[nElems];
-        ASSERT_EQ(AF_SUCCESS, af_get_data_ptr((void*)goldData, goldArray));
+        std::vector<T> goldData(nElems);
+        ASSERT_EQ(AF_SUCCESS, af_get_data_ptr((void*)goldData.data(), goldArray));
 
-        ASSERT_EQ(true, compareArraysRMSD(nElems, goldData, outData, 0.02f));
+        ASSERT_EQ(true, compareArraysRMSD(nElems, goldData.data(), outData.data(), 0.02f));
 
         ASSERT_EQ(AF_SUCCESS, af_release_array(inArray));
         ASSERT_EQ(AF_SUCCESS, af_release_array(outArray));
@@ -103,25 +103,23 @@ void bilateralDataTest(string pTestFile)
     af::dim4 dims      = numDims[0];
     af_array outArray  = 0;
     af_array inArray   = 0;
-    outType *outData;
 
     ASSERT_EQ(AF_SUCCESS, af_create_array(&inArray, &(in[0].front()),
                 dims.ndims(), dims.get(), (af_dtype)af::dtype_traits<inType>::af_type));
 
     ASSERT_EQ(AF_SUCCESS, af_bilateral(&outArray, inArray, 2.25f, 25.56f, false));
 
-    outData = new outType[dims.elements()];
+    std::vector<outType> outData(dims.elements());
 
-    ASSERT_EQ(AF_SUCCESS, af_get_data_ptr((void*)outData, outArray));
+    ASSERT_EQ(AF_SUCCESS, af_get_data_ptr((void*)outData.data(), outArray));
 
     for (size_t testIter=0; testIter<tests.size(); ++testIter) {
         vector<outType> currGoldBar = tests[testIter];
         size_t nElems = currGoldBar.size();
-        ASSERT_EQ(true, compareArraysRMSD(nElems, &currGoldBar.front(), outData, 0.02f));
+        ASSERT_EQ(true, compareArraysRMSD(nElems, &currGoldBar.front(), outData.data(), 0.02f));
     }
 
     // cleanup
-    delete[] outData;
     ASSERT_EQ(AF_SUCCESS, af_release_array(inArray));
     ASSERT_EQ(AF_SUCCESS, af_release_array(outArray));
 }
@@ -171,17 +169,14 @@ TEST(Bilateral, CPP)
     array a(dims, &(in[0].front()));
     array b = af::bilateral(a, 2.25f, 25.56f, false);
 
-    float *outData = new float[dims.elements()];
-    b.host(outData);
+    std::vector<float> outData(dims.elements());
+    b.host(outData.data());
 
     for (size_t testIter=0; testIter<tests.size(); ++testIter) {
         vector<float> currGoldBar = tests[testIter];
         size_t nElems = currGoldBar.size();
-        ASSERT_EQ(true, compareArraysRMSD(nElems, &currGoldBar.front(), outData, 0.02f));
+        ASSERT_EQ(true, compareArraysRMSD(nElems, currGoldBar.data(), outData.data(), 0.02f));
     }
-
-    // cleanup
-    delete[] outData;
 }
 
 
diff --git a/test/canny.cpp b/test/canny.cpp
index 16dd5d5171..19e159edaa 100644
--- a/test/canny.cpp
+++ b/test/canny.cpp
@@ -51,9 +51,9 @@ void cannyTest(string pTestFile)
 
     ASSERT_EQ(AF_SUCCESS, af_canny(&outArray, sArray, AF_MANUAL_THRESHOLD, 0.4147f, 0.8454f, 3, true));
 
-    char *outData = new char[sDims.elements()];
+    std::vector<char> outData(sDims.elements());
 
-    ASSERT_EQ(AF_SUCCESS, af_get_data_ptr((void*)outData, outArray));
+    ASSERT_EQ(AF_SUCCESS, af_get_data_ptr((void*)outData.data(), outArray));
 
     vector<char> currGoldBar = tests[0];
     size_t nElems        = currGoldBar.size();
@@ -62,7 +62,6 @@ void cannyTest(string pTestFile)
     }
 
     // cleanup
-    delete[] outData;
     ASSERT_EQ(AF_SUCCESS, af_release_array(sArray));
     ASSERT_EQ(AF_SUCCESS, af_release_array(outArray));
 }
@@ -110,13 +109,13 @@ void cannyImageOtsuTest(string pTestFile, bool isColor)
 
         ASSERT_EQ(AF_SUCCESS, af_canny(&outArray, inArray, AF_AUTO_OTSU_THRESHOLD, 0.08, 0.32, 3, false));
 
-        char * outData = new char[nElems];
-        ASSERT_EQ(AF_SUCCESS, af_get_data_ptr((void*)outData, outArray));
+        std::vector<char> outData(nElems);
+        ASSERT_EQ(AF_SUCCESS, af_get_data_ptr((void*)outData.data(), outArray));
 
-        char * goldData= new char[nElems];
-        ASSERT_EQ(AF_SUCCESS, af_get_data_ptr((void*)goldData, goldArray));
+        std::vector<char> goldData(nElems);
+        ASSERT_EQ(AF_SUCCESS, af_get_data_ptr((void*)goldData.data(), goldArray));
 
-        ASSERT_EQ(true, compareArraysRMSD(nElems, goldData, outData, 1.0e-3));
+        ASSERT_EQ(true, compareArraysRMSD(nElems, goldData.data(), outData.data(), 1.0e-3));
 
         ASSERT_EQ(AF_SUCCESS, af_release_array(inArray));
         ASSERT_EQ(AF_SUCCESS, af_release_array(outArray));
diff --git a/test/covariance.cpp b/test/covariance.cpp
index 933f617612..224003dfb7 100644
--- a/test/covariance.cpp
+++ b/test/covariance.cpp
@@ -96,17 +96,14 @@ void covTest(string pFileName, bool isbiased=false)
     vector<outType> currGoldBar(tests[0].begin(), tests[0].end());
 
     size_t nElems    = currGoldBar.size();
-    outType *outData = new outType[nElems];
+    std::vector<outType> outData(nElems);
 
-    c.host((void*)outData);
+    c.host((void*)outData.data());
 
     for (size_t elIter=0; elIter<nElems; ++elIter) {
         ASSERT_NEAR(::real(currGoldBar[elIter]), ::real(outData[elIter]), 1.0e-3)<< "at: " << elIter<< std::endl;
         ASSERT_NEAR(::imag(currGoldBar[elIter]), ::imag(outData[elIter]), 1.0e-3)<< "at: " << elIter<< std::endl;
     }
-
-    // cleanup
-    delete[] outData;
 }
 
 TYPED_TEST(Covariance, Vector)
diff --git a/test/gen_assign.cpp b/test/gen_assign.cpp
index 3e950c3d46..70cecf7ae3 100644
--- a/test/gen_assign.cpp
+++ b/test/gen_assign.cpp
@@ -59,15 +59,14 @@ void testGeneralAssignOneArray(string pTestFile, const dim_t ndims, af_index_t*
 
     vector<float> currGoldBar = tests[0];
     size_t nElems = currGoldBar.size();
-    float *outData = new float[nElems];
+    std::vector<float> outData(nElems);
 
-    ASSERT_EQ(AF_SUCCESS, af_get_data_ptr((void*)outData, outArray));
+    ASSERT_EQ(AF_SUCCESS, af_get_data_ptr((void*)outData.data(), outArray));
 
     for (size_t elIter=0; elIter<nElems; ++elIter) {
         ASSERT_EQ(currGoldBar[elIter], outData[elIter])<< "at: " << elIter<< std::endl;
     }
 
-    delete[] outData;
     ASSERT_EQ(AF_SUCCESS, af_release_array(rhsArray));
     ASSERT_EQ(AF_SUCCESS, af_release_array(lhsArray));
     ASSERT_EQ(AF_SUCCESS, af_release_array(idxArray));
@@ -124,15 +123,14 @@ TEST(GeneralAssign, SSSS)
 
     vector<float> currGoldBar = tests[0];
     size_t nElems = currGoldBar.size();
-    float *outData = new float[nElems];
+    std::vector<float> outData(nElems);
 
-    ASSERT_EQ(AF_SUCCESS, af_get_data_ptr((void*)outData, outArray));
+    ASSERT_EQ(AF_SUCCESS, af_get_data_ptr((void*)outData.data(), outArray));
 
     for (size_t elIter=0; elIter<nElems; ++elIter) {
         ASSERT_EQ(currGoldBar[elIter], outData[elIter])<< "at: " << elIter<< std::endl;
     }
 
-    delete[] outData;
     ASSERT_EQ(AF_SUCCESS, af_release_array(rhsArray));
     ASSERT_EQ(AF_SUCCESS, af_release_array(lhsArray));
     ASSERT_EQ(AF_SUCCESS, af_release_array(outArray));
@@ -192,18 +190,21 @@ TEST(GeneralAssign, AAAA)
 
     vector<float> currGoldBar = tests[0];
     size_t nElems = currGoldBar.size();
-    float *outData = new float[nElems];
+    std::vector<float> outData(nElems);
 
-    ASSERT_EQ(AF_SUCCESS, af_get_data_ptr((void*)outData, outArray));
+    ASSERT_EQ(AF_SUCCESS, af_get_data_ptr((void*)outData.data(), outArray));
 
     for (size_t elIter=0; elIter<nElems; ++elIter) {
         ASSERT_EQ(currGoldBar[elIter], outData[elIter])<< "at: " << elIter<< std::endl;
     }
 
-    delete[] outData;
     ASSERT_EQ(AF_SUCCESS, af_release_array(rhsArray));
     ASSERT_EQ(AF_SUCCESS, af_release_array(lhsArray));
     ASSERT_EQ(AF_SUCCESS, af_release_array(outArray));
+    ASSERT_EQ(AF_SUCCESS, af_release_array(idxArray0));
+    ASSERT_EQ(AF_SUCCESS, af_release_array(idxArray1));
+    ASSERT_EQ(AF_SUCCESS, af_release_array(idxArray2));
+    ASSERT_EQ(AF_SUCCESS, af_release_array(idxArray3));
 }
 
 
diff --git a/test/gen_index.cpp b/test/gen_index.cpp
index 7fcd3fd46e..7b4638e25b 100644
--- a/test/gen_index.cpp
+++ b/test/gen_index.cpp
@@ -54,15 +54,14 @@ void testGeneralIndexOneArray(string pTestFile, const dim_t ndims, af_index_t* i
 
     vector<float> currGoldBar = tests[0];
     size_t nElems = currGoldBar.size();
-    float *outData = new float[nElems];
+    std::vector<float> outData(nElems);
 
-    ASSERT_EQ(AF_SUCCESS, af_get_data_ptr((void*)outData, outArray));
+    ASSERT_EQ(AF_SUCCESS, af_get_data_ptr((void*)outData.data(), outArray));
 
     for (size_t elIter=0; elIter<nElems; ++elIter) {
         ASSERT_EQ(currGoldBar[elIter], outData[elIter])<< "at: " << elIter<< std::endl;
     }
 
-    delete[] outData;
     ASSERT_EQ(AF_SUCCESS, af_release_array(inArray));
     ASSERT_EQ(AF_SUCCESS, af_release_array(idxArray));
     ASSERT_EQ(AF_SUCCESS, af_release_array(outArray));
@@ -141,15 +140,14 @@ TEST(GeneralIndex, AASS)
 
     vector<float> currGoldBar = tests[0];
     size_t nElems = currGoldBar.size();
-    float *outData = new float[nElems];
+    std::vector<float> outData(nElems);
 
-    ASSERT_EQ(AF_SUCCESS, af_get_data_ptr((void*)outData, outArray));
+    ASSERT_EQ(AF_SUCCESS, af_get_data_ptr((void*)outData.data(), outArray));
 
     for (size_t elIter=0; elIter<nElems; ++elIter) {
         ASSERT_EQ(currGoldBar[elIter], outData[elIter])<< "at: " << elIter<< std::endl;
     }
 
-    delete[] outData;
     ASSERT_EQ(AF_SUCCESS, af_release_array(inArray));
     ASSERT_EQ(AF_SUCCESS, af_release_array(idxArray0));
     ASSERT_EQ(AF_SUCCESS, af_release_array(idxArray1));
diff --git a/test/histogram.cpp b/test/histogram.cpp
index c83ba0464f..992e998e29 100644
--- a/test/histogram.cpp
+++ b/test/histogram.cpp
@@ -48,14 +48,14 @@ void histTest(string pTestFile, unsigned nbins, double minval, double maxval)
 
     af_array outArray   = 0;
     af_array inArray    = 0;
-    outType *outData;
+
     ASSERT_EQ(AF_SUCCESS, af_create_array(&inArray, &(in[0].front()), dims.ndims(), dims.get(), (af_dtype) af::dtype_traits<inType>::af_type));
 
     ASSERT_EQ(AF_SUCCESS,af_histogram(&outArray,inArray,nbins,minval,maxval));
 
-    outData = new outType[dims.elements()];
+    std::vector<outType> outData(dims.elements());
 
-    ASSERT_EQ(AF_SUCCESS, af_get_data_ptr((void*)outData, outArray));
+    ASSERT_EQ(AF_SUCCESS, af_get_data_ptr((void*)outData.data(), outArray));
 
     for (size_t testIter=0; testIter<tests.size(); ++testIter) {
         vector<outType> currGoldBar = tests[testIter];
@@ -66,7 +66,6 @@ void histTest(string pTestFile, unsigned nbins, double minval, double maxval)
     }
 
     // cleanup
-    delete[] outData;
     ASSERT_EQ(AF_SUCCESS, af_release_array(inArray));
     ASSERT_EQ(AF_SUCCESS, af_release_array(outArray));
 }
@@ -118,8 +117,8 @@ TEST(Histogram, CPP)
     af::array output = histogram(input, nbins, minval, maxval);
 //! [hist_nominmax]
 
-    uint *outData = new uint[output.elements()];
-    output.host((void*)outData);
+    std::vector<uint> outData(output.elements());
+    output.host((void*)outData.data());
 
     for (size_t testIter=0; testIter<tests.size(); ++testIter) {
         vector<uint> currGoldBar = tests[testIter];
@@ -128,9 +127,6 @@ TEST(Histogram, CPP)
             ASSERT_EQ(currGoldBar[elIter],outData[elIter])<< "at: " << elIter<< std::endl;
         }
     }
-
-    // cleanup
-    delete[] outData;
 }
 
 /////////////////////////////////// Documentation Snippets //////////////////////////////////
diff --git a/test/hsv_rgb.cpp b/test/hsv_rgb.cpp
index 5e221c2e6d..cd057b0152 100644
--- a/test/hsv_rgb.cpp
+++ b/test/hsv_rgb.cpp
@@ -45,17 +45,14 @@ TEST(hsv2rgb, CPP)
     af::array input(dims, &(in[0].front()));
     af::array output = af::hsv2rgb(input);
 
-    float *outData = new float[dims.elements()];
-    output.host((void*)outData);
+    std::vector<float> outData(dims.elements());
+    output.host((void*)outData.data());
 
     vector<float> currGoldBar = tests[0];
     size_t nElems = currGoldBar.size();
     for (size_t elIter=0; elIter<nElems; ++elIter) {
         ASSERT_NEAR(currGoldBar[elIter], outData[elIter], 1.0e-3)<< "at: " << elIter<< std::endl;
     }
-
-    // cleanup
-    delete[] outData;
 }
 
 TEST(rgb2hsv, CPP)
@@ -70,15 +67,12 @@ TEST(rgb2hsv, CPP)
     af::array input(dims, &(in[0].front()));
     af::array output = af::rgb2hsv(input);
 
-    float *outData = new float[dims.elements()];
-    output.host((void*)outData);
+    std::vector<float> outData(dims.elements());
+    output.host((void*)outData.data());
 
     vector<float> currGoldBar = tests[0];
     size_t nElems = currGoldBar.size();
     for (size_t elIter=0; elIter<nElems; ++elIter) {
         ASSERT_NEAR(currGoldBar[elIter], outData[elIter], 1.0e-3)<< "at: " << elIter<< std::endl;
     }
-
-    // cleanup
-    delete[] outData;
 }
diff --git a/test/match_template.cpp b/test/match_template.cpp
index adebea4ac1..cfa23e7847 100644
--- a/test/match_template.cpp
+++ b/test/match_template.cpp
@@ -57,9 +57,9 @@ void matchTemplateTest(string pTestFile, af_match_type pMatchType)
 
     ASSERT_EQ(AF_SUCCESS, af_match_template(&outArray, sArray, tArray, pMatchType));
 
-    outType *outData = new outType[sDims.elements()];
+    std::vector<outType> outData(sDims.elements());
 
-    ASSERT_EQ(AF_SUCCESS, af_get_data_ptr((void*)outData, outArray));
+    ASSERT_EQ(AF_SUCCESS, af_get_data_ptr((void*)outData.data(), outArray));
 
     vector<outType> currGoldBar = tests[0];
     size_t nElems        = currGoldBar.size();
@@ -68,7 +68,6 @@ void matchTemplateTest(string pTestFile, af_match_type pMatchType)
     }
 
     // cleanup
-    delete[] outData;
     ASSERT_EQ(AF_SUCCESS, af_release_array(sArray));
     ASSERT_EQ(AF_SUCCESS, af_release_array(tArray));
     ASSERT_EQ(AF_SUCCESS, af_release_array(outArray));
diff --git a/test/mean.cpp b/test/mean.cpp
index e3f7031747..dd370f4e93 100644
--- a/test/mean.cpp
+++ b/test/mean.cpp
@@ -92,9 +92,9 @@ void meanDimTest(string pFileName, dim_t dim, bool isWeighted=false)
 
         af::array outArray = af::mean(inArray, dim);
 
-        outType *outData = new outType[dims.elements()];
+        std::vector<outType> outData(dims.elements());
 
-        outArray.host((void*)outData);
+        outArray.host((void*)outData.data());
 
         vector<outType> currGoldBar(tests[0].begin(), tests[0].end());
         size_t nElems = currGoldBar.size();
@@ -102,9 +102,6 @@ void meanDimTest(string pFileName, dim_t dim, bool isWeighted=false)
             ASSERT_NEAR(::real(currGoldBar[elIter]), ::real(outData[elIter]), 1.0e-3)<< "at: " << elIter<< std::endl;
             ASSERT_NEAR(::imag(currGoldBar[elIter]), ::imag(outData[elIter]), 1.0e-3)<< "at: " << elIter<< std::endl;
         }
-
-        // cleanup
-        delete[] outData;
     } else {
         af::dim4 dims  = numDims[0];
         af::dim4 wdims = numDims[1];
@@ -116,9 +113,9 @@ void meanDimTest(string pFileName, dim_t dim, bool isWeighted=false)
 
         af::array outArray = af::mean(inArray, wtsArray, dim);
 
-        outType *outData = new outType[dims.elements()];
+        std::vector<outType> outData(dims.elements());
 
-        outArray.host((void*)outData);
+        outArray.host((void*)outData.data());
 
         vector<outType> currGoldBar(tests[0].begin(), tests[0].end());
         size_t nElems = currGoldBar.size();
@@ -126,9 +123,6 @@ void meanDimTest(string pFileName, dim_t dim, bool isWeighted=false)
             ASSERT_NEAR(::real(currGoldBar[elIter]), ::real(outData[elIter]), 1.0e-3)<< "at: " << elIter<< std::endl;
             ASSERT_NEAR(::imag(currGoldBar[elIter]), ::imag(outData[elIter]), 1.0e-3)<< "at: " << elIter<< std::endl;
         }
-
-        // cleanup
-        delete[] outData;
     }
 }
 
diff --git a/test/meanshift.cpp b/test/meanshift.cpp
index a35ca288d9..c9fe41af29 100644
--- a/test/meanshift.cpp
+++ b/test/meanshift.cpp
@@ -84,13 +84,13 @@ void meanshiftTest(string pTestFile)
 
         ASSERT_EQ(AF_SUCCESS, af_mean_shift(&outArray, inArray, 2.25f, 25.56f, 5, isColor));
 
-        T * outData = new T[nElems];
-        ASSERT_EQ(AF_SUCCESS, af_get_data_ptr((void*)outData, outArray));
+        std::vector<T> outData(nElems);
+        ASSERT_EQ(AF_SUCCESS, af_get_data_ptr((void*)outData.data(), outArray));
 
-        T * goldData= new T[nElems];
-        ASSERT_EQ(AF_SUCCESS, af_get_data_ptr((void*)goldData, goldArray));
+        std::vector<T> goldData(nElems);
+        ASSERT_EQ(AF_SUCCESS, af_get_data_ptr((void*)goldData.data(), goldArray));
 
-        ASSERT_EQ(true, compareArraysRMSD(nElems, goldData, outData, 0.07f));
+        ASSERT_EQ(true, compareArraysRMSD(nElems, goldData.data(), outData.data(), 0.07f));
 
         ASSERT_EQ(AF_SUCCESS, af_release_array(inArray));
         ASSERT_EQ(AF_SUCCESS, af_release_array(inArray_f32));
@@ -145,16 +145,13 @@ TEST(Meanshift, Color_CPP)
         dim_t nElems = gold.elements();
         af::array output= af::meanShift(img, 2.25f, 25.56f, 5, true);
 
-        float * outData = new float[nElems];
-        output.host((void*)outData);
+        std::vector<float> outData(nElems);
+        output.host((void*)outData.data());
 
-        float * goldData= new float[nElems];
-        gold.host((void*)goldData);
+        std::vector<float> goldData(nElems);
+        gold.host((void*)goldData.data());
 
-        ASSERT_EQ(true, compareArraysRMSD(nElems, goldData, outData, 0.07f));
-        // cleanup
-        delete[] outData;
-        delete[] goldData;
+        ASSERT_EQ(true, compareArraysRMSD(nElems, goldData.data(), outData.data(), 0.07f));
     }
 }
 
diff --git a/test/medfilt.cpp b/test/medfilt.cpp
index 1296e22b0c..e41e7a45cb 100644
--- a/test/medfilt.cpp
+++ b/test/medfilt.cpp
@@ -60,9 +60,9 @@ void medfiltTest(string pTestFile, dim_t w_len, dim_t w_wid, af_border_type pad)
 
     ASSERT_EQ(AF_SUCCESS, af_medfilt2(&outArray, inArray, w_len, w_wid, pad));
 
-    T *outData = new T[dims.elements()];
+    std::vector<T> outData(dims.elements());
 
-    ASSERT_EQ(AF_SUCCESS, af_get_data_ptr((void*)outData, outArray));
+    ASSERT_EQ(AF_SUCCESS, af_get_data_ptr((void*)outData.data(), outArray));
 
     vector<T> currGoldBar = tests[0];
     size_t nElems        = currGoldBar.size();
@@ -71,7 +71,6 @@ void medfiltTest(string pTestFile, dim_t w_len, dim_t w_wid, af_border_type pad)
     }
 
     // cleanup
-    delete[] outData;
     ASSERT_EQ(AF_SUCCESS, af_release_array(inArray));
     ASSERT_EQ(AF_SUCCESS, af_release_array(outArray));
 }
@@ -117,9 +116,9 @@ void medfilt1_Test(string pTestFile, dim_t w_wid, af_border_type pad)
 
     ASSERT_EQ(AF_SUCCESS, af_medfilt1(&outArray, inArray, w_wid, pad));
 
-    T *outData = new T[dims.elements()];
+    std::vector<T> outData(dims.elements());
 
-    ASSERT_EQ(AF_SUCCESS, af_get_data_ptr((void*)outData, outArray));
+    ASSERT_EQ(AF_SUCCESS, af_get_data_ptr((void*)outData.data(), outArray));
 
     vector<T> currGoldBar = tests[0];
     size_t nElems        = currGoldBar.size();
@@ -128,7 +127,6 @@ void medfilt1_Test(string pTestFile, dim_t w_wid, af_border_type pad)
     }
 
     // cleanup
-    delete[] outData;
     ASSERT_EQ(AF_SUCCESS, af_release_array(inArray));
     ASSERT_EQ(AF_SUCCESS, af_release_array(outArray));
 }
@@ -186,13 +184,13 @@ void medfiltImageTest(string pTestFile, dim_t w_len, dim_t w_wid)
 
         ASSERT_EQ(AF_SUCCESS, af_medfilt2(&outArray, inArray, w_len, w_wid, AF_PAD_ZERO));
 
-        T * outData = new T[nElems];
-        ASSERT_EQ(AF_SUCCESS, af_get_data_ptr((void*)outData, outArray));
+        std::vector<T> outData(nElems);
+        ASSERT_EQ(AF_SUCCESS, af_get_data_ptr((void*)outData.data(), outArray));
 
-        T * goldData= new T[nElems];
-        ASSERT_EQ(AF_SUCCESS, af_get_data_ptr((void*)goldData, goldArray));
+        std::vector<T> goldData(nElems);
+        ASSERT_EQ(AF_SUCCESS, af_get_data_ptr((void*)goldData.data(), goldArray));
 
-        ASSERT_EQ(true, compareArraysRMSD(nElems, goldData, outData, 0.018f));
+        ASSERT_EQ(true, compareArraysRMSD(nElems, goldData.data(), outData.data(), 0.018f));
 
         ASSERT_EQ(AF_SUCCESS, af_release_array(inArray));
         ASSERT_EQ(AF_SUCCESS, af_release_array(outArray));
@@ -224,6 +222,7 @@ void medfiltInputTest(void)
     ASSERT_EQ(true, medfilt1);
 
     ASSERT_EQ(AF_SUCCESS, af_release_array(inArray));
+    ASSERT_EQ(AF_SUCCESS, af_release_array(outArray));
 }
 
 TYPED_TEST(MedianFilter, InvalidArray)
@@ -360,17 +359,14 @@ TEST(MedianFilter, CPP)
     af::array input(dims, &(in[0].front()));
     af::array output = af::medfilt(input, w_len, w_wid, AF_PAD_SYM);
 
-    float *outData = new float[dims.elements()];
-    output.host((void*)outData);
+    std::vector<float> outData(dims.elements());
+    output.host((void*)outData.data());
 
     vector<float> currGoldBar = tests[0];
     size_t nElems = currGoldBar.size();
     for (size_t elIter=0; elIter<nElems; ++elIter) {
         ASSERT_EQ(currGoldBar[elIter], outData[elIter])<< "at: " << elIter<< std::endl;
     }
-
-    // cleanup
-    delete[] outData;
 }
 
 TEST(MedianFilter1d, CPP)
@@ -390,17 +386,14 @@ TEST(MedianFilter1d, CPP)
     af::array input(dims, &(in[0].front()));
     af::array output = af::medfilt1(input, w_wid, AF_PAD_SYM);
 
-    float *outData = new float[dims.elements()];
-    output.host((void*)outData);
+    std::vector<float> outData(dims.elements());
+    output.host((void*)outData.data());
 
     vector<float> currGoldBar = tests[0];
     size_t nElems = currGoldBar.size();
     for (size_t elIter=0; elIter<nElems; ++elIter) {
         ASSERT_EQ(currGoldBar[elIter], outData[elIter])<< "at: " << elIter<< std::endl;
     }
-
-    // cleanup
-    delete[] outData;
 }
 
 
diff --git a/test/morph.cpp b/test/morph.cpp
index 8aa0885cc4..11499b6ca5 100644
--- a/test/morph.cpp
+++ b/test/morph.cpp
@@ -49,7 +49,6 @@ void morphTest(string pTestFile)
     af_array outArray  = 0;
     af_array inArray   = 0;
     af_array maskArray = 0;
-    inType *outData;
 
     ASSERT_EQ(AF_SUCCESS, af_create_array(&inArray, &(in[0].front()),
                 dims.ndims(), dims.get(), (af_dtype)af::dtype_traits<inType>::af_type));
@@ -69,9 +68,9 @@ void morphTest(string pTestFile)
             ASSERT_EQ(AF_SUCCESS, af_erode(&outArray, inArray, maskArray));
     }
 
-    outData = new inType[dims.elements()];
+    std::vector<inType> outData(dims.elements());
 
-    ASSERT_EQ(AF_SUCCESS, af_get_data_ptr((void*)outData, outArray));
+    ASSERT_EQ(AF_SUCCESS, af_get_data_ptr((void*)outData.data(), outArray));
 
     for (size_t testIter=0; testIter<tests.size(); ++testIter) {
         vector<inType> currGoldBar = tests[testIter];
@@ -82,7 +81,6 @@ void morphTest(string pTestFile)
     }
 
     // cleanup
-    delete[] outData;
     ASSERT_EQ(AF_SUCCESS, af_release_array(inArray));
     ASSERT_EQ(AF_SUCCESS, af_release_array(maskArray));
     ASSERT_EQ(AF_SUCCESS, af_release_array(outArray));
@@ -159,13 +157,13 @@ void morphImageTest(string pTestFile)
         else
             ASSERT_EQ(AF_SUCCESS, af_erode(&outArray, inArray, maskArray));
 
-        T * outData = new T[nElems];
-        ASSERT_EQ(AF_SUCCESS, af_get_data_ptr((void*)outData, outArray));
+        std::vector<T> outData(nElems);
+        ASSERT_EQ(AF_SUCCESS, af_get_data_ptr((void*)outData.data(), outArray));
 
-        T * goldData= new T[nElems];
-        ASSERT_EQ(AF_SUCCESS, af_get_data_ptr((void*)goldData, goldArray));
+        std::vector<T> goldData(nElems);
+        ASSERT_EQ(AF_SUCCESS, af_get_data_ptr((void*)goldData.data(), goldArray));
 
-        ASSERT_EQ(true, compareArraysRMSD(nElems, goldData, outData, 0.018f));
+        ASSERT_EQ(true, compareArraysRMSD(nElems, goldData.data(), outData.data(), 0.018f));
 
         ASSERT_EQ(AF_SUCCESS, af_release_array(inArray));
         ASSERT_EQ(AF_SUCCESS, af_release_array(maskArray));
@@ -371,16 +369,13 @@ void cppMorphImageTest(string pTestFile)
         else
             output = erode(img, mask);
 
-        T * outData = new T[nElems];
-        output.host((void*)outData);
+        std::vector<T> outData(nElems);
+        output.host((void*)outData.data());
 
-        T * goldData= new T[nElems];
-        gold.host((void*)goldData);
+        std::vector<T> goldData(nElems);
+        gold.host((void*)goldData.data());
 
-        ASSERT_EQ(true, compareArraysRMSD(nElems, goldData, outData, 0.018f));
-        //cleanup
-        delete[] outData;
-        delete[] goldData;
+        ASSERT_EQ(true, compareArraysRMSD(nElems, goldData.data(), outData.data(), 0.018f));
     }
 }
 
@@ -452,8 +447,8 @@ TEST(Morph, EdgeIssue1564)
     array dilated = dilate(input.as(b8), mask.as(b8));
 
     size_t nElems = dilated.elements();
-    char * outData = new char[nElems];
-    dilated.host((void*)outData);
+    std::vector<char> outData(nElems);
+    dilated.host((void*)outData.data());
 
     for (size_t i=0; i<nElems; ++i) {
         ASSERT_EQ((int)outData[i], goldData[i]);
diff --git a/test/sobel.cpp b/test/sobel.cpp
index d3f4528af0..69a6250a8c 100644
--- a/test/sobel.cpp
+++ b/test/sobel.cpp
@@ -61,11 +61,11 @@ void testSobelDerivatives(string pTestFile)
 
     ASSERT_EQ(AF_SUCCESS, af_sobel_operator(&dxArray, &dyArray, inArray, 3));
 
-    To *dxData = new To[dims.elements()];
-    To *dyData = new To[dims.elements()];
+    std::vector<To> dxData(dims.elements());
+    std::vector<To> dyData(dims.elements());
 
-    ASSERT_EQ(AF_SUCCESS, af_get_data_ptr((void*)dxData, dxArray));
-    ASSERT_EQ(AF_SUCCESS, af_get_data_ptr((void*)dyData, dyArray));
+    ASSERT_EQ(AF_SUCCESS, af_get_data_ptr((void*)dxData.data(), dxArray));
+    ASSERT_EQ(AF_SUCCESS, af_get_data_ptr((void*)dyData.data(), dyArray));
 
     vector<To> currDXGoldBar = tests[0];
     vector<To> currDYGoldBar = tests[1];
@@ -79,8 +79,6 @@ void testSobelDerivatives(string pTestFile)
     }
 
     // cleanup
-    delete[] dxData;
-    delete[] dyData;
     ASSERT_EQ(AF_SUCCESS, af_release_array(inArray));
     ASSERT_EQ(AF_SUCCESS, af_release_array(dxArray));
     ASSERT_EQ(AF_SUCCESS, af_release_array(dyArray));
diff --git a/test/stdev.cpp b/test/stdev.cpp
index ff70948752..c8f61e4364 100644
--- a/test/stdev.cpp
+++ b/test/stdev.cpp
@@ -93,17 +93,14 @@ void stdevDimTest(string pFileName, dim_t dim=-1)
     vector<outType> currGoldBar(tests[0].begin(), tests[0].end());
 
     size_t nElems    = currGoldBar.size();
-    outType *outData = new outType[nElems];
+    std::vector<outType> outData(nElems);
 
-    b.host((void*)outData);
+    b.host((void*)outData.data());
 
     for (size_t elIter=0; elIter<nElems; ++elIter) {
         ASSERT_NEAR(::real(currGoldBar[elIter]), ::real(outData[elIter]), 1.0e-3)<< "at: " << elIter<< std::endl;
         ASSERT_NEAR(::imag(currGoldBar[elIter]), ::imag(outData[elIter]), 1.0e-3)<< "at: " << elIter<< std::endl;
     }
-
-    // cleanup
-    delete[] outData;
 }
 
 TYPED_TEST(StandardDev, Dim0)
@@ -160,17 +157,14 @@ void stdevDimIndexTest(string pFileName, dim_t dim=-1)
     vector<outType> currGoldBar(tests[0].begin(), tests[0].end());
 
     size_t nElems    = currGoldBar.size();
-    outType *outData = new outType[nElems];
+    std::vector<outType> outData(nElems);
 
-    c.host((void*)outData);
+    c.host((void*)outData.data());
 
     for (size_t elIter=0; elIter<nElems; ++elIter) {
         ASSERT_NEAR(::real(currGoldBar[elIter]), ::real(outData[elIter]), 1.0e-3)<< "at: " << elIter<< std::endl;
         ASSERT_NEAR(::imag(currGoldBar[elIter]), ::imag(outData[elIter]), 1.0e-3)<< "at: " << elIter<< std::endl;
     }
-
-    // cleanup
-    delete[] outData;
 }
 
 TYPED_TEST(StandardDev, IndexedArrayDim0)
diff --git a/test/susan.cpp b/test/susan.cpp
index 259a319ce7..ee846d1c4d 100644
--- a/test/susan.cpp
+++ b/test/susan.cpp
@@ -81,19 +81,20 @@ void susanTest(string pTestFile, float t, float g)
 
         af::features out = af::susan(in, 3, t, g, 0.05f, 3);
 
-        float * outX           = new float[gold[0].size()];
-        float * outY           = new float[gold[1].size()];
-        float * outScore       = new float[gold[2].size()];
-        float * outOrientation = new float[gold[3].size()];
-        float * outSize        = new float[gold[4].size()];
-        out.getX().host(outX);
-        out.getY().host(outY);
-        out.getScore().host(outScore);
-        out.getOrientation().host(outOrientation);
-        out.getSize().host(outSize);
+        std::vector<float> outX          (gold[0].size());
+        std::vector<float> outY          (gold[1].size());
+        std::vector<float> outScore      (gold[2].size());
+        std::vector<float> outOrientation(gold[3].size());
+        std::vector<float> outSize       (gold[4].size());
+        out.getX().host(outX.data());
+        out.getY().host(outY.data());
+        out.getScore().host(outScore.data());
+        out.getOrientation().host(outOrientation.data());
+        out.getSize().host(outSize.data());
 
         vector<feat_t> out_feat;
-        array_to_feat(out_feat, outX, outY, outScore, outOrientation, outSize, out.getNumFeatures());
+        array_to_feat(out_feat, outX.data(), outY.data(), outScore.data(),
+                      outOrientation.data(), outSize.data(), out.getNumFeatures());
 
         vector<feat_t> gold_feat;
         array_to_feat(gold_feat, &gold[0].front(), &gold[1].front(), &gold[2].front(), &gold[3].front(), &gold[4].front(), gold[0].size());
@@ -108,12 +109,6 @@ void susanTest(string pTestFile, float t, float g)
             ASSERT_EQ(out_feat[elIter].f[3], gold_feat[elIter].f[3]) << "at: " << elIter << std::endl;
             ASSERT_EQ(out_feat[elIter].f[4], gold_feat[elIter].f[4]) << "at: " << elIter << std::endl;
         }
-
-        delete [] outX;
-        delete [] outY;
-        delete [] outScore;
-        delete [] outOrientation;
-        delete [] outSize;
     }
 }
 
diff --git a/test/ycbcr_rgb.cpp b/test/ycbcr_rgb.cpp
index f2c7aacf79..4f4d7cec73 100644
--- a/test/ycbcr_rgb.cpp
+++ b/test/ycbcr_rgb.cpp
@@ -45,17 +45,14 @@ TEST(ycbcr2rgb, CPP)
     af::array input(dims, &(in[0].front()));
     af::array output = af::ycbcr2rgb(input);
 
-    float *outData = new float[dims.elements()];
-    output.host((void*)outData);
+    std::vector<float> outData(dims.elements());
+    output.host(outData.data());
 
     vector<float> currGoldBar = tests[0];
     size_t nElems = currGoldBar.size();
     for (size_t elIter=0; elIter<nElems; ++elIter) {
         ASSERT_NEAR(currGoldBar[elIter], outData[elIter], 1.0e-3)<< "at: " << elIter<< std::endl;
     }
-
-    // cleanup
-    delete[] outData;
 }
 
 TEST(rgb2ycbcr, CPP)
@@ -70,15 +67,12 @@ TEST(rgb2ycbcr, CPP)
     af::array input(dims, &(in[0].front()));
     af::array output = af::rgb2ycbcr(input);
 
-    float *outData = new float[dims.elements()];
-    output.host((void*)outData);
+    std::vector<float> outData(dims.elements());
+    output.host(outData.data());
 
     vector<float> currGoldBar = tests[0];
     size_t nElems = currGoldBar.size();
     for (size_t elIter=0; elIter<nElems; ++elIter) {
         ASSERT_NEAR(currGoldBar[elIter], outData[elIter], 1.0e-3)<< "at: " << elIter<< std::endl;
     }
-
-    // cleanup
-    delete[] outData;
 }

From 657c1eae63ae3ead35bb8440113fb8e76bfd0002 Mon Sep 17 00:00:00 2001
From: pradeep <pradeep@arrayfire.com>
Date: Wed, 26 Apr 2017 18:58:32 +0530
Subject: [PATCH 75/82] Fix memory scope tests in threading unit tests

Disable memory management stress test for CPU backend.
---
 test/threading.cpp | 77 +++++++++++++++++-----------------------------
 1 file changed, 28 insertions(+), 49 deletions(-)

diff --git a/test/threading.cpp b/test/threading.cpp
index 60d088e79d..40f82414eb 100644
--- a/test/threading.cpp
+++ b/test/threading.cpp
@@ -14,6 +14,7 @@
 #include <thread>
 #include <complex>
 #include <chrono>
+#include <condition_variable>
 #include <thread>
 #include <iterator>
 #include <vector>
@@ -198,20 +199,34 @@ static void cleanSlate()
     ASSERT_EQ(af::getMemStepSize(), step_bytes);
 }
 
+std::condition_variable cv;
+std::mutex cvMutex;
+size_t counter = THREAD_COUNT;
+
 void doubleAllocationTest()
 {
     af::setDevice(0);
 
+    //Block until all threads are launched and the
+    //counter variable hits zero
+    std::unique_lock<std::mutex> lock(cvMutex);
+    //Check for current thread launch counter value
+    //if reached zero, notify others to continue
+    //otherwise block current thread
+    if (--counter==0)
+        cv.notify_all();
+    else
+        cv.wait(lock, [] {return counter==0;});
+    lock.unlock();
+
     af::array a = randu(5, 5);
 
-    for (int i = 0; i < 100; ++i)
-    {
-        a = randu(5, 5);
-        std::this_thread::sleep_for(std::chrono::milliseconds(10));
-    }
+    // Wait for for other threads to hit randu call
+    // while this thread's variable a is still in scope.
+    std::this_thread::sleep_for(std::chrono::seconds(2));
 }
 
-TEST(Threading, MemoryManagement_Double_Alloc)
+TEST(Threading, MemoryManagementScope)
 {
     cleanSlate(); // Clean up everything done so far
 
@@ -232,48 +247,8 @@ TEST(Threading, MemoryManagement_Double_Alloc)
 
     ASSERT_EQ( lock_buffers,     0u);
     ASSERT_EQ(  lock_bytes,      0u);
-    ASSERT_LE(alloc_buffers,    64u);
-    ASSERT_GT(alloc_buffers,    32u);
-    ASSERT_LE(  alloc_bytes, 65536u);
-    ASSERT_GT(  alloc_bytes, 32768u);
-}
-
-void singleAllocationTest()
-{
-    af::setDevice(0);
-
-    for (int i = 0; i < 100; ++i)
-    {
-        af::array a = af::randu(5, 5);
-        std::this_thread::sleep_for(std::chrono::milliseconds(10));
-    }
-}
-
-TEST(Threading, MemoryManagement_Single_Alloc)
-{
-    cleanSlate(); // Clean up everything done so far
-
-    vector<std::thread> tests;
-
-    for (int t=0; t<THREAD_COUNT; ++t)
-        tests.emplace_back(singleAllocationTest);
-
-    for (int t=0; t<THREAD_COUNT; ++t)
-        if (tests[t].joinable())
-            tests[t].join();
-
-    size_t alloc_bytes, alloc_buffers;
-    size_t lock_bytes, lock_buffers;
-
-    af::deviceMemInfo(&alloc_bytes, &alloc_buffers,
-            &lock_bytes, &lock_buffers);
-
-    ASSERT_EQ( lock_buffers,     0u);
-    ASSERT_EQ(  lock_bytes,      0u);
-    ASSERT_LE(alloc_buffers,    32u);
-    ASSERT_GT(alloc_buffers,     0u);
-    ASSERT_LE(  alloc_bytes, 32768u);
-    ASSERT_GT(  alloc_bytes,     0u);
+    ASSERT_EQ(alloc_buffers,    32u);
+    ASSERT_EQ(  alloc_bytes, 32768u);
 }
 
 void jitAllocationTest()
@@ -703,7 +678,10 @@ TEST(Threading, Sparse)
             tests[testId].join();
 }
 
-TEST(Threading, MemoryManagerStressTest) {
+//FIXME Disable for CPU backend
+#if !defined(AF_CPU)
+TEST(Threading, MemoryManagerStressTest)
+{
   vector<std::thread> threads;
   for (int i = 0; i < THREAD_COUNT; i++) {
     threads.emplace_back([] {
@@ -734,3 +712,4 @@ TEST(Threading, MemoryManagerStressTest) {
     t.join();
   }
 }
+#endif

From 9154174bc2861c17e49f9f50092ff72d7277f3b5 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Sat, 29 Apr 2017 17:36:55 -0400
Subject: [PATCH 76/82] Make sparse_common function static. c++11 flag for
 single file test

---
 test/CMakeLists.txt    |  1 +
 test/sparse_common.hpp | 14 +++++++-------
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 031af068a3..38d5cc7f3d 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -75,6 +75,7 @@ MACRO(CREATE_TESTS BACKEND AFLIBNAME GTEST_LIBS OTHER_LIBS)
       SET(TEST_NAME_BASIC test_basic_${BACKEND})
       ADD_EXECUTABLE(${TEST_NAME} ${CPP_FILES})
       ADD_EXECUTABLE(${TEST_NAME_BASIC} basic_c.c)
+      SET_TARGET_PROPERTIES(${TEST_NAME} PROPERTIES COMPILE_OPTIONS -std=c++11)
 
       TARGET_LINK_LIBRARIES(${TEST_NAME}  PRIVATE ${AFLIBNAME}
         PRIVATE ${THREAD_LIB_FLAG}
diff --git a/test/sparse_common.hpp b/test/sparse_common.hpp
index e8a8efc32e..34cfd12ae3 100644
--- a/test/sparse_common.hpp
+++ b/test/sparse_common.hpp
@@ -28,7 +28,7 @@ using af::cdouble;
 ///////////////////////////////// CPP ////////////////////////////////////
 //
 
-template<typename T>
+template<typename T> static
 af::array makeSparse(af::array A, int factor)
 {
     A = floor(A * 1000);
@@ -62,12 +62,12 @@ af::array makeSparse<cdouble>(af::array A, int factor)
     return A;
 }
 
-double calc_norm(af::array lhs, af::array rhs)
+static double calc_norm(af::array lhs, af::array rhs)
 {
     return af::max<double>(af::abs(lhs - rhs) / (af::abs(lhs) + af::abs(rhs) + 1E-5));
 }
 
-template<typename T>
+template<typename T> static
 void sparseTester(const int m, const int n, const int k, int factor, double eps,
                   int targetDevice=-1)
 {
@@ -102,7 +102,7 @@ void sparseTester(const int m, const int n, const int k, int factor, double eps,
     ASSERT_NEAR(0, calc_norm(imag(dRes1), imag(sRes1)), eps);
 }
 
-template<typename T>
+template<typename T> static
 void sparseTransposeTester(const int m, const int n, const int k, int factor, double eps,
                            int targetDevice=-1)
 {
@@ -142,7 +142,7 @@ void sparseTransposeTester(const int m, const int n, const int k, int factor, do
     ASSERT_NEAR(0, calc_norm(imag(dRes3), imag(sRes3)), eps);
 }
 
-template<typename T>
+template<typename T> static
 void convertCSR(const int M, const int N, const float ratio, int targetDevice=-1)
 {
     if (targetDevice>=0)
@@ -164,7 +164,7 @@ void convertCSR(const int M, const int N, const float ratio, int targetDevice=-1
 
 // This test essentially verifies that the sparse structures have the correct
 // dimensions and indices using a very basic test
-template<af_storage stype>
+template<af_storage stype> static
 void createFunction()
 {
     af::array in = af::sparse(af::identity(3, 3), stype);
@@ -181,7 +181,7 @@ void createFunction()
     ASSERT_EQ(0, af::max<int   >(colIdx - af::range(af::dim4(colIdx.elements()), 0, s32)));
 }
 
-template<typename Ti, typename To>
+template<typename Ti, typename To> static
 void sparseCastTester(const int m, const int n, int factor)
 {
     if (noDoubleTests<Ti>()) return;

From b86e5dff00f9039762078413ed293fe13b0e459d Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Sat, 29 Apr 2017 20:01:46 -0400
Subject: [PATCH 77/82] Disable memory stress test

---
 test/threading.cpp | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/test/threading.cpp b/test/threading.cpp
index 40f82414eb..cf82960f3c 100644
--- a/test/threading.cpp
+++ b/test/threading.cpp
@@ -678,9 +678,7 @@ TEST(Threading, Sparse)
             tests[testId].join();
 }
 
-//FIXME Disable for CPU backend
-#if !defined(AF_CPU)
-TEST(Threading, MemoryManagerStressTest)
+TEST(Threading, DISABLED_MemoryManagerStressTest)
 {
   vector<std::thread> threads;
   for (int i = 0; i < THREAD_COUNT; i++) {
@@ -712,4 +710,3 @@ TEST(Threading, MemoryManagerStressTest)
     t.join();
   }
 }
-#endif

From 5f53f6bd82e198f9bd92e1814a980e0b1bb69cee Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Sat, 29 Apr 2017 22:59:17 -0400
Subject: [PATCH 78/82] Added missing headers

---
 src/backend/common/MemoryManager.hpp | 12 +++++++-----
 src/backend/opencl/platform.cpp      |  5 +++--
 2 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/src/backend/common/MemoryManager.hpp b/src/backend/common/MemoryManager.hpp
index 219385ffd2..b2138da488 100644
--- a/src/backend/common/MemoryManager.hpp
+++ b/src/backend/common/MemoryManager.hpp
@@ -9,15 +9,17 @@
 
 #pragma once
 
-#include <vector>
-#include <mutex>
-#include <unordered_map>
-#include <iostream>
-#include <iomanip>
 #include <err_common.hpp>
 #include <dispatch.hpp>
 #include <util.hpp>
 
+#include <algorithm>
+#include <iomanip>
+#include <iostream>
+#include <mutex>
+#include <unordered_map>
+#include <vector>
+
 namespace common
 {
 typedef std::recursive_mutex mutex_t;
diff --git a/src/backend/opencl/platform.cpp b/src/backend/opencl/platform.cpp
index 652abc2c93..b88c100927 100644
--- a/src/backend/opencl/platform.cpp
+++ b/src/backend/opencl/platform.cpp
@@ -36,13 +36,14 @@
 
 #include <algorithm>
 #include <cctype>
+#include <cstring>
 #include <functional>
 #include <map>
 #include <mutex>
-#include <string>
 #include <sstream>
 #include <stdexcept>
-#include <cstring>
+#include <string>
+#include <thread>
 #include <utility>
 #include <vector>
 

From 9ba6a9657d1bf064b08dea570401bc500f4bf6a9 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Sun, 30 Apr 2017 12:04:22 -0400
Subject: [PATCH 79/82] Fix narrowing error/warning in canny

---
 src/api/c/canny.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/api/c/canny.cpp b/src/api/c/canny.cpp
index b32dfee6a1..b6b4dbe93c 100644
--- a/src/api/c/canny.cpp
+++ b/src/api/c/canny.cpp
@@ -173,7 +173,7 @@ template<typename T>
 af_array cannyHelper(const Array<T> in, const float t1, const af_canny_threshold ct,
                      const float t2, const unsigned sw, const bool isf)
 {
-    static const std::vector<float> v = {-0.11021, -0.23691, -0.30576, -0.23691, -0.11021};
+    static const std::vector<float> v = {-0.11021f, -0.23691f, -0.30576f, -0.23691f, -0.11021f};
     Array<float> cFilter= detail::createHostDataArray<float>(dim4(5, 1), v.data());
     Array<float> rFilter= detail::createHostDataArray<float>(dim4(1, 5), v.data());
 

From 77edd69dd88718030f667e81498f2dcc08562269 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Mon, 1 May 2017 13:12:21 -0400
Subject: [PATCH 80/82] Fix headers in the CUDA backend

---
 src/backend/cuda/jit.cpp      |  1 +
 src/backend/cuda/platform.cpp | 17 ++++++++++-------
 2 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/src/backend/cuda/jit.cpp b/src/backend/cuda/jit.cpp
index 776c5c8f28..8341731289 100644
--- a/src/backend/cuda/jit.cpp
+++ b/src/backend/cuda/jit.cpp
@@ -46,6 +46,7 @@
 #include <map>
 #include <memory>
 #include <stdexcept>
+#include <thread>
 #include <vector>
 
 namespace cuda
diff --git a/src/backend/cuda/platform.cpp b/src/backend/cuda/platform.cpp
index 89cc827758..48a32bf96e 100644
--- a/src/backend/cuda/platform.cpp
+++ b/src/backend/cuda/platform.cpp
@@ -14,17 +14,20 @@
 #include <util.hpp>
 #include <version.hpp>
 #include <driver.h>
-#include <vector>
-#include <string>
-#include <algorithm>
-#include <sstream>
-#include <stdexcept>
-#include <cstdio>
-#include <cstring>
 #include <err_cuda.hpp>
 #include <util.hpp>
 #include <host_memory.hpp>
 
+#include <algorithm>
+#include <cstdio>
+#include <cstring>
+#include <mutex>
+#include <sstream>
+#include <stdexcept>
+#include <string>
+#include <thread>
+#include <vector>
+
 using namespace std;
 
 namespace cuda

From 913e1abdf385c3afb950898c38ee4286fc21343f Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Mon, 1 May 2017 15:31:24 -0400
Subject: [PATCH 81/82] Remove static storage qualifier from thread_local
 variables

---
 src/backend/cuda/jit.cpp        |  2 +-
 src/backend/cuda/platform.cpp   | 18 +++++++++---------
 src/backend/opencl/platform.cpp |  6 +++---
 3 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/src/backend/cuda/jit.cpp b/src/backend/cuda/jit.cpp
index 8341731289..097bc120cf 100644
--- a/src/backend/cuda/jit.cpp
+++ b/src/backend/cuda/jit.cpp
@@ -562,7 +562,7 @@ static CUfunction getKernel(const vector<Node *> &output_nodes,
 {
     typedef std::map<std::string, kc_entry_t> kc_t;
 
-    thread_local static kc_t kernelCaches[DeviceManager::MAX_DEVICES];
+    thread_local kc_t kernelCaches[DeviceManager::MAX_DEVICES];
 
     string funcName = getFuncName(output_nodes, full_nodes, full_ids, is_linear);
     int device      = getActiveDeviceId();
diff --git a/src/backend/cuda/platform.cpp b/src/backend/cuda/platform.cpp
index 48a32bf96e..b80d288f91 100644
--- a/src/backend/cuda/platform.cpp
+++ b/src/backend/cuda/platform.cpp
@@ -285,7 +285,7 @@ unsigned getMaxJitSize()
 
 int& tlocalActiveDeviceId()
 {
-    thread_local static int activeDeviceId = 0;
+    thread_local int activeDeviceId = 0;
 
     return activeDeviceId;
 }
@@ -432,15 +432,15 @@ GraphicsResourceManager& interopManager()
 
 PlanCache& fftManager()
 {
-    thread_local static PlanCache cufftManagers[DeviceManager::MAX_DEVICES];
+    thread_local PlanCache cufftManagers[DeviceManager::MAX_DEVICES];
 
     return cufftManagers[getActiveDeviceId()];
 }
 
 BlasHandle blasHandle()
 {
-    thread_local static std::unique_ptr<cublasHandle> cublasHandles[DeviceManager::MAX_DEVICES];
-    thread_local static std::once_flag initFlags[DeviceManager::MAX_DEVICES];
+    thread_local std::unique_ptr<cublasHandle> cublasHandles[DeviceManager::MAX_DEVICES];
+    thread_local std::once_flag initFlags[DeviceManager::MAX_DEVICES];
 
     int id = cuda::getActiveDeviceId();
 
@@ -453,8 +453,8 @@ BlasHandle blasHandle()
 
 SolveHandle solverDnHandle()
 {
-    thread_local static std::unique_ptr<cusolverDnHandle> cusolverHandles[DeviceManager::MAX_DEVICES];
-    thread_local static std::once_flag initFlags[DeviceManager::MAX_DEVICES];
+    thread_local std::unique_ptr<cusolverDnHandle> cusolverHandles[DeviceManager::MAX_DEVICES];
+    thread_local std::once_flag initFlags[DeviceManager::MAX_DEVICES];
 
     int id = cuda::getActiveDeviceId();
 
@@ -480,8 +480,8 @@ SolveHandle solverDnHandle()
 
 SparseHandle sparseHandle()
 {
-    thread_local static std::unique_ptr<cusparseHandle> cusparseHandles[DeviceManager::MAX_DEVICES];
-    thread_local static std::once_flag initFlags[DeviceManager::MAX_DEVICES];
+    thread_local std::unique_ptr<cusparseHandle> cusparseHandles[DeviceManager::MAX_DEVICES];
+    thread_local std::once_flag initFlags[DeviceManager::MAX_DEVICES];
 
     int id = cuda::getActiveDeviceId();
 
@@ -553,7 +553,7 @@ void DeviceManager::sortDevices(sort_mode mode)
 
 int DeviceManager::setActiveDevice(int device, int nId)
 {
-    thread_local static bool retryFlag = true;
+    thread_local bool retryFlag = true;
 
     int numDevices = cuDevices.size();
 
diff --git a/src/backend/opencl/platform.cpp b/src/backend/opencl/platform.cpp
index b88c100927..dc7e16b7b4 100644
--- a/src/backend/opencl/platform.cpp
+++ b/src/backend/opencl/platform.cpp
@@ -280,7 +280,7 @@ std::pair<unsigned, unsigned>& tlocalActiveDeviceId()
 {
     // First element is active context id
     // Second element is active queue id
-    thread_local static device_id_t activeDeviceId(0, 0);
+    thread_local device_id_t activeDeviceId(0, 0);
 
     return activeDeviceId;
 }
@@ -713,14 +713,14 @@ GraphicsResourceManager& interopManager()
 
 PlanCache& fftManager()
 {
-    thread_local static PlanCache clfftManagers[DeviceManager::MAX_DEVICES];
+    thread_local PlanCache clfftManagers[DeviceManager::MAX_DEVICES];
 
     return clfftManagers[getActiveDeviceId()];
 }
 
 kc_t& getKernelCache(int device)
 {
-    thread_local static kc_t kernelCaches[DeviceManager::MAX_DEVICES];
+    thread_local kc_t kernelCaches[DeviceManager::MAX_DEVICES];
 
     return kernelCaches[device];
 }

From d2907650f42f76d665cc31fffce1de34c52eeed5 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Wed, 17 May 2017 16:30:51 -0400
Subject: [PATCH 82/82] Move solve threading tests to solve_dense.cpp

---
 test/CMakeLists.txt         |  2 +-
 test/manual_memory_test.cpp | 16 -----------
 test/memory.cpp             | 20 -------------
 test/memory_lock.cpp        | 20 -------------
 test/solve_common.hpp       | 27 ++++++++++++++----
 test/solve_dense.cpp        | 40 ++++++++++++++++++++++++++
 test/testHelpers.hpp        | 23 +++++++++++++++
 test/threading.cpp          | 57 -------------------------------------
 8 files changed, 85 insertions(+), 120 deletions(-)

diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 38d5cc7f3d..ed123c5781 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -121,7 +121,7 @@ MACRO(CREATE_TESTS BACKEND AFLIBNAME GTEST_LIBS OTHER_LIBS)
           COMPILE_FLAGS -DAF_${DEF_NAME}
           FOLDER "Tests/${BACKEND}")
 
-        IF (${FNAME} STREQUAL "threading")
+        IF (${FNAME} MATCHES "threading|solve")
             SET_TARGET_PROPERTIES(${TEST_NAME} PROPERTIES COMPILE_OPTIONS -std=c++11)
         ENDIF()
 
diff --git a/test/manual_memory_test.cpp b/test/manual_memory_test.cpp
index 95011ac8f1..3f6fa1ac47 100644
--- a/test/manual_memory_test.cpp
+++ b/test/manual_memory_test.cpp
@@ -21,22 +21,6 @@ using std::string;
 using std::cout;
 using std::endl;
 
-static void cleanSlate()
-{
-    size_t alloc_bytes, alloc_buffers;
-    size_t lock_bytes, lock_buffers;
-
-    af::deviceGC();
-
-    af::deviceMemInfo(&alloc_bytes, &alloc_buffers,
-                      &lock_bytes, &lock_buffers);
-
-    ASSERT_EQ(alloc_buffers, 0u);
-    ASSERT_EQ(lock_buffers, 0u);
-    ASSERT_EQ(alloc_bytes, 0u);
-    ASSERT_EQ(lock_bytes, 0u);
-}
-
 TEST(Memory, recover)
 {
     cleanSlate(); // Clean up everything done so far
diff --git a/test/memory.cpp b/test/memory.cpp
index 5342781b7f..d5d15dd37b 100644
--- a/test/memory.cpp
+++ b/test/memory.cpp
@@ -26,26 +26,6 @@ using af::cdouble;
 
 const size_t step_bytes = 1024;
 
-static void cleanSlate()
-{
-    size_t alloc_bytes, alloc_buffers;
-    size_t lock_bytes, lock_buffers;
-
-    af::deviceGC();
-
-    af::deviceMemInfo(&alloc_bytes, &alloc_buffers,
-                      &lock_bytes, &lock_buffers);
-
-    ASSERT_EQ(alloc_buffers, 0u);
-    ASSERT_EQ(lock_buffers, 0u);
-    ASSERT_EQ(alloc_bytes, 0u);
-    ASSERT_EQ(lock_bytes, 0u);
-
-    af::setMemStepSize(step_bytes);
-
-    ASSERT_EQ(af::getMemStepSize(), step_bytes);
-}
-
 TEST(Memory, Scope)
 {
     size_t alloc_bytes, alloc_buffers;
diff --git a/test/memory_lock.cpp b/test/memory_lock.cpp
index 6a4fe8665a..63331f0d03 100644
--- a/test/memory_lock.cpp
+++ b/test/memory_lock.cpp
@@ -23,26 +23,6 @@ using std::endl;
 
 const size_t step_bytes = 1024;
 
-static void cleanSlate()
-{
-    size_t alloc_bytes, alloc_buffers;
-    size_t lock_bytes, lock_buffers;
-
-    af::deviceGC();
-
-    af::deviceMemInfo(&alloc_bytes, &alloc_buffers,
-                      &lock_bytes, &lock_buffers);
-
-    ASSERT_EQ(alloc_buffers, 0u);
-    ASSERT_EQ(lock_buffers, 0u);
-    ASSERT_EQ(alloc_bytes, 0u);
-    ASSERT_EQ(lock_bytes, 0u);
-
-    af::setMemStepSize(step_bytes);
-
-    ASSERT_EQ(af::getMemStepSize(), step_bytes);
-}
-
 // This test should be by itself as it leaks memory intentionally
 TEST(Memory, lock)
 {
diff --git a/test/solve_common.hpp b/test/solve_common.hpp
index 3149a7b3f3..c532cad42c 100644
--- a/test/solve_common.hpp
+++ b/test/solve_common.hpp
@@ -56,8 +56,13 @@ void solveTester(const int m, const int n, const int k, double eps, int targetDe
     af::array B1 = af::matmul(A, X1);
     //! [ex_solve_recon]
 
-    ASSERT_NEAR(0, af::sum<double>(af::abs(real(B0 - B1))) / (m * k), eps);
-    ASSERT_NEAR(0, af::sum<double>(af::abs(imag(B0 - B1))) / (m * k), eps);
+    if(noDoubleTests<T>()) {
+        ASSERT_NEAR(0, af::sum<double>(af::abs(real(B0 - B1))) / (m * k), eps);
+        ASSERT_NEAR(0, af::sum<double>(af::abs(imag(B0 - B1))) / (m * k), eps);
+    } else {
+        ASSERT_NEAR(0, af::sum<float>(af::abs(real(B0 - B1))) / (m * k), eps);
+        ASSERT_NEAR(0, af::sum<float>(af::abs(imag(B0 - B1))) / (m * k), eps);
+    }
 }
 
 template<typename T>
@@ -88,8 +93,13 @@ void solveLUTester(const int n, const int k, double eps, int targetDevice=-1)
 
     af::array B1 = af::matmul(A, X1);
 
-    ASSERT_NEAR(0, af::sum<double>(af::abs(real(B0 - B1))) / (n * k), eps);
-    ASSERT_NEAR(0, af::sum<double>(af::abs(imag(B0 - B1))) / (n * k), eps);
+    if(noDoubleTests<T>()) {
+        ASSERT_NEAR(0, af::sum<double>(af::abs(real(B0 - B1))) / (n * k), eps);
+        ASSERT_NEAR(0, af::sum<double>(af::abs(imag(B0 - B1))) / (n * k), eps);
+    } else {
+        ASSERT_NEAR(0, af::sum<float>(af::abs(real(B0 - B1))) / (n * k), eps);
+        ASSERT_NEAR(0, af::sum<float>(af::abs(imag(B0 - B1))) / (n * k), eps);
+    }
 }
 
 template<typename T>
@@ -134,6 +144,11 @@ void solveTriangleTester(const int n, const int k, bool is_upper, double eps, in
 
     af::array B1 = af::matmul(AT, X1);
 
-    ASSERT_NEAR(0, af::sum<double>(af::abs(real(B0 - B1))) / (n * k), eps);
-    ASSERT_NEAR(0, af::sum<double>(af::abs(imag(B0 - B1))) / (n * k), eps);
+    if(noDoubleTests<T>()) {
+        ASSERT_NEAR(0, af::sum<double>(af::abs(real(B0 - B1))) / (n * k), eps);
+        ASSERT_NEAR(0, af::sum<double>(af::abs(imag(B0 - B1))) / (n * k), eps);
+    } else {
+        ASSERT_NEAR(0, af::sum<float>(af::abs(real(B0 - B1))) / (n * k), eps);
+        ASSERT_NEAR(0, af::sum<float>(af::abs(imag(B0 - B1))) / (n * k), eps);
+    }
 }
diff --git a/test/solve_dense.cpp b/test/solve_dense.cpp
index 641b05c7e3..3caa656336 100644
--- a/test/solve_dense.cpp
+++ b/test/solve_dense.cpp
@@ -10,6 +10,7 @@
 #include <gtest/gtest.h>
 #include <testHelpers.hpp>
 #include "solve_common.hpp"
+#include <thread>
 
 #define SOLVE_LU_TESTS(T, eps)                          \
     TEST(SOLVE_LU, T##Reg)                              \
@@ -80,4 +81,43 @@ SOLVE_TESTS(double, 1E-5)
 SOLVE_TESTS(cfloat, 0.01)
 SOLVE_TESTS(cdouble, 1E-5)
 
+
+#if !defined(AF_OPENCL)
+int nextTargetDeviceId()
+{
+  static int nextId = 0;
+  return nextId++;
+}
+
+#define SOLVE_LU_TESTS_THREADING(T, eps)                                                                          \
+    tests.emplace_back(solveLUTester<T>, 1000, 100, eps, nextTargetDeviceId()%numDevices);              \
+    tests.emplace_back(solveTriangleTester<T>, 1000, 100, true, eps, nextTargetDeviceId()%numDevices);  \
+    tests.emplace_back(solveTriangleTester<T>, 1000, 100, false, eps, nextTargetDeviceId()%numDevices); \
+    tests.emplace_back(solveTester<T>, 1000, 1000, 100, eps, nextTargetDeviceId()%numDevices);          \
+    tests.emplace_back(solveTester<T>, 800, 1000, 200, eps, nextTargetDeviceId()%numDevices);           \
+    tests.emplace_back(solveTester<T>, 800, 600, 64, eps, nextTargetDeviceId()%numDevices);             \
+
+TEST(SOLVE, Threading)
+{
+    cleanSlate(); // Clean up everything done so far
+
+    vector<std::thread> tests;
+
+    int numDevices = 1;
+    ASSERT_EQ(AF_SUCCESS, af_get_device_count(&numDevices));
+
+    SOLVE_LU_TESTS_THREADING(float, 0.01);
+    SOLVE_LU_TESTS_THREADING(cfloat, 0.01);
+    if (noDoubleTests<double>()) {
+        SOLVE_LU_TESTS_THREADING(double, 1E-5);
+        SOLVE_LU_TESTS_THREADING(cdouble, 1E-5);
+    }
+
+    for (size_t testId=0; testId<tests.size(); ++testId)
+        if (tests[testId].joinable())
+            tests[testId].join();
+}
+
+#undef SOLVE_LU_TESTS_THREADING
+#endif
 #undef SOLVE_TESTS
diff --git a/test/testHelpers.hpp b/test/testHelpers.hpp
index 999a71b97d..58b820eabd 100644
--- a/test/testHelpers.hpp
+++ b/test/testHelpers.hpp
@@ -6,6 +6,7 @@
  * The complete license agreement can be obtained at:
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
+#pragma once
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wunused-function"
 
@@ -453,4 +454,26 @@ af::array cpu_randu(const af::dim4 dims)
     return af::array(dims, (T *)&out[0]);
 }
 
+void cleanSlate()
+{
+  const size_t step_bytes = 1024;
+
+  size_t alloc_bytes, alloc_buffers;
+  size_t lock_bytes, lock_buffers;
+
+  af::deviceGC();
+
+  af::deviceMemInfo(&alloc_bytes, &alloc_buffers,
+                    &lock_bytes, &lock_buffers);
+
+  ASSERT_EQ(alloc_buffers, 0u);
+  ASSERT_EQ(lock_buffers, 0u);
+  ASSERT_EQ(alloc_bytes, 0u);
+  ASSERT_EQ(lock_bytes, 0u);
+
+  af::setMemStepSize(step_bytes);
+
+  ASSERT_EQ(af::getMemStepSize(), step_bytes);
+}
+
 #pragma GCC diagnostic pop
diff --git a/test/threading.cpp b/test/threading.cpp
index cf82960f3c..01a130fb99 100644
--- a/test/threading.cpp
+++ b/test/threading.cpp
@@ -19,7 +19,6 @@
 #include <iterator>
 #include <vector>
 #include <testHelpers.hpp>
-#include <solve_common.hpp>
 #include <sparse_common.hpp>
 
 using namespace af;
@@ -177,28 +176,6 @@ TEST(Threading, SimultaneousRead)
             tests[t].join();
 }
 
-static void cleanSlate()
-{
-    const size_t step_bytes = 1024;
-
-    size_t alloc_bytes, alloc_buffers;
-    size_t lock_bytes, lock_buffers;
-
-    af::deviceGC();
-
-    af::deviceMemInfo(&alloc_bytes, &alloc_buffers,
-                      &lock_bytes, &lock_buffers);
-
-    ASSERT_EQ(alloc_buffers, 0u);
-    ASSERT_EQ(lock_buffers, 0u);
-    ASSERT_EQ(alloc_bytes, 0u);
-    ASSERT_EQ(lock_bytes, 0u);
-
-    af::setMemStepSize(step_bytes);
-
-    ASSERT_EQ(af::getMemStepSize(), step_bytes);
-}
-
 std::condition_variable cv;
 std::mutex cvMutex;
 size_t counter = THREAD_COUNT;
@@ -615,40 +592,6 @@ TEST(Threading, BLAS)
             tests[testId].join();
 }
 
-#if !defined(AF_OPENCL)
-
-#define SOLVE_LU_TESTS(T, eps)                                                                          \
-    tests.emplace_back(solveLUTester<T>, 1000, 100, eps, nextTargetDeviceId()%numDevices);              \
-    tests.emplace_back(solveTriangleTester<T>, 1000, 100, true, eps, nextTargetDeviceId()%numDevices);  \
-    tests.emplace_back(solveTriangleTester<T>, 1000, 100, false, eps, nextTargetDeviceId()%numDevices); \
-    tests.emplace_back(solveTester<T>, 1000, 1000, 100, eps, nextTargetDeviceId()%numDevices);          \
-    tests.emplace_back(solveTester<T>, 800, 1000, 200, eps, nextTargetDeviceId()%numDevices);           \
-    tests.emplace_back(solveTester<T>, 800, 600, 64, eps, nextTargetDeviceId()%numDevices);             \
-
-TEST(Threading, SolveDense)
-{
-    cleanSlate(); // Clean up everything done so far
-
-    vector<std::thread> tests;
-
-    int numDevices = 1;
-    ASSERT_EQ(AF_SUCCESS, af_get_device_count(&numDevices));
-
-    SOLVE_LU_TESTS(float, 0.01);
-    SOLVE_LU_TESTS(cfloat, 0.01);
-    if (noDoubleTests<double>()) {
-        SOLVE_LU_TESTS(double, 1E-5);
-        SOLVE_LU_TESTS(cdouble, 1E-5);
-    }
-
-    for (size_t testId=0; testId<tests.size(); ++testId)
-        if (tests[testId].joinable())
-            tests[testId].join();
-}
-
-#undef SOLVE_LU_TESTS
-#endif
-
 #define SPARSE_TESTS(T, eps)                                                    \
   tests.emplace_back(sparseTester<T>, 1000, 1000, 100, 5, eps, nextTargetDeviceId()%numDevices);  \
   tests.emplace_back(sparseTester<T>, 500, 1000, 250, 1, eps, nextTargetDeviceId()%numDevices);   \